diff --git a/.Rbuildignore b/.Rbuildignore index 24393d5..08e91e5 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -7,12 +7,14 @@ ^docs$ ^pkgdown$ ^\.github$ +^\.lintr$ ^codecov\.yml$ ^\.Renviron$ ^\.Renviron\.local$ ^data-raw$ ^dev$ ^PINBOARD\.md$ +^SPEC-measurement-layer\.md$ ^sessions$ ^tools$ ^vignettes/articles$ diff --git a/.gitignore b/.gitignore index 5524130..a091cae 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,8 @@ Thumbs.db docs/ # Test artifacts -tests/testthat/_snaps/ +tests/testthat/_snaps/* +!tests/testthat/_snaps/codebook.md # Environment .Renviron diff --git a/.lintr b/.lintr new file mode 100644 index 0000000..222d7bf --- /dev/null +++ b/.lintr @@ -0,0 +1,37 @@ +linters: linters_with_defaults( + line_length_linter = line_length_linter(120L), + object_usage_linter = NULL, + commented_code_linter = NULL + ) +exclusions: list( + "data-raw/record-doc-outputs.R", + "R/agents.R", + "R/audio.R", + "R/auth.R", + "R/batches.R", + "R/chat.R", + "R/check.R", + "R/config.R", + "R/content-safety-extra.R", + "R/conversations.R", + "R/embed-batch.R", + "R/embed.R", + "R/evals.R", + "R/groundedness.R", + "R/image.R", + "R/moderate.R", + "R/responses.R", + "R/schema.R", + "R/tidymodels.R", + "R/token-azure-identity.R", + "R/utils.R", + "R/vector-stores.R", + "tests/manual/test_all_examples.R", + "tests/manual/test_contracts_2026_h2.R", + "tests/testthat/helper.R", + "tests/testthat/test-agents.R", + "tests/testthat/test-groundedness.R", + "tests/testthat/test-image.R", + "vignettes/embeddings.Rmd" + ) +encoding: "UTF-8" diff --git a/DESCRIPTION b/DESCRIPTION index f275401..a3f0f34 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -20,6 +20,7 @@ RoxygenNote: 7.3.3 Imports: cli, curl, + digest, dplyr, httr2 (>= 1.0.0), jsonlite, diff --git a/NAMESPACE b/NAMESPACE index 34414b0..8efc11d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,12 +1,15 @@ # Generated by roxygen2: do not edit by hand +S3method(format,foundry_codebook) S3method(generics::tidy,step_foundry_embed) +S3method(print,foundry_codebook) S3method(print,step_foundry_embed) S3method(recipes::bake,step_foundry_embed) S3method(recipes::prep,step_foundry_embed) S3method(recipes::required_pkgs,step_foundry_embed) export("%>%") export(as_foundry_schema) +export(codebook_diff) export(foundry_agent) export(foundry_agent_create) export(foundry_agent_delete) @@ -35,6 +38,7 @@ export(foundry_blocklists) export(foundry_cache_clear) export(foundry_chat) export(foundry_check_setup) +export(foundry_codebook) export(foundry_consistency) export(foundry_conversation_create) export(foundry_conversation_delete) @@ -136,6 +140,10 @@ export(schema_number) export(schema_object) export(schema_string) export(step_foundry_embed) +export(type_boolean) +export(type_enum) +export(type_number) +export(type_string) importFrom(lifecycle,deprecated) importFrom(magrittr,"%>%") importFrom(rlang,"%||%") diff --git a/NEWS.md b/NEWS.md index e154cd9..af9b218 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ Initial development release of foundryR, a tidy interface to Microsoft Azure AI - Added cloud evaluation workflows with grader constructors (`foundry_grader_string_check()`, `foundry_grader_text_similarity()`, `foundry_grader_label_model()`, `foundry_grader_score_model()`, and `foundry_grader_azure_ai()` for `builtin.*` evaluators), evaluation and run lifecycle functions (`foundry_eval_create()`, `foundry_evals()`, `foundry_eval_get()`, `foundry_eval_delete()`, `foundry_eval_run_create()`, `foundry_eval_runs()`, `foundry_eval_run_get()`, `foundry_eval_run_cancel()`), and `foundry_eval_run_output_items()`, which returns per-row grader scores as a tibble (roadmap 2026 H2). - Added preview Content Safety operations: `foundry_protected_code()` for protected-material-in-code detection, `foundry_moderate_multimodal()` for image-with-text moderation, and `foundry_task_adherence()` (with `foundry_agent_tool()`, `foundry_agent_tool_call()`, and `foundry_agent_message()` builders) for agent task-adherence checks (roadmap 2026 H2). - Added Responses API conversation and vector store helpers, including `foundry_conversation_create()`, `foundry_conversations()`, `foundry_vector_store_create()`, `foundry_vector_search()`, and `foundry_tool_file_search()` (roadmap 2026 H2). +- Added `foundry_codebook()` and `codebook_diff()` for versioned measurement-layer codebooks with deterministic SHA-256 hashes, schema helper wrappers, print output, and codebook diffs (measurement layer M1). - Added schema constructors with `foundry_schema()`, `schema_string()`, `schema_enum()`, `schema_number()`, `schema_integer()`, `schema_boolean()`, `schema_array()`, `schema_object()`, and `as_foundry_schema()` for strict structured-output schemas (roadmap 2026 H2). - Added validation helpers `foundry_agreement()`, `foundry_consistency()`, and `foundry_provenance()` for publication-oriented annotation checks and reproducibility metadata (roadmap 2026 H2). - Added v1 Batch API workflows with `foundry_batch_create()`, `foundry_batches()`, `foundry_batch_get()`, `foundry_batch_cancel()`, and `foundry_batch_requests()` for large-scale prompt, annotation, extraction, and classification jobs. diff --git a/R/codebook.R b/R/codebook.R new file mode 100644 index 0000000..e8d266f --- /dev/null +++ b/R/codebook.R @@ -0,0 +1,383 @@ +#' Create a measurement codebook +#' +#' A codebook records the instructions, JSON Schema, examples, semantic +#' version, creation time, and deterministic SHA-256 hash for an LLM annotation +#' instrument. The hash is computed from a canonical JSON serialization of +#' `instructions`, `schema`, `examples`, and `version`, in that order. Before +#' serialization, schema arrays are preserved with the same internal helper used +#' by structured outputs so single-value `enum` and `required` arrays do not +#' collapse to scalars. The payload is serialized with +#' `jsonlite::toJSON(auto_unbox = TRUE, digits = NA, null = "null")`, +#' normalized with `enc2utf8()`, and hashed with SHA-256. +#' +#' @param name Character. Lowercase slug for the codebook; hyphens are allowed. +#' @param version Character. Semantic version string. +#' @param instructions Character. System or instruction prompt for annotation. +#' @param schema List. JSON Schema object, typically from [foundry_schema()]. +#' @param examples List or `NULL`. Few-shot examples included in the codebook +#' hash. +#' +#' @return A `foundry_codebook` object. +#' @export +#' +#' @examples +#' \dontrun{ +#' codebook <- foundry_codebook( +#' name = "ai-applicability", +#' version = "1.0.0", +#' instructions = "Label whether the task could use AI assistance.", +#' schema = foundry_schema( +#' ai_applicable = type_boolean("AI could materially assist the task") +#' ), +#' examples = list( +#' list(text = "Draft a memo", ai_applicable = TRUE), +#' list(text = "Lift a heavy box", ai_applicable = FALSE) +#' ) +#' ) +#' } +foundry_codebook <- function(name, + version, + instructions, + schema, + examples = NULL) { + foundry_check_codebook_name(name) + foundry_check_semver(version) + foundry_check_character_scalar(instructions, "instructions") + schema <- as_foundry_schema(schema) + if (!is.null(examples) && !is.list(examples)) { + cli::cli_abort("{.arg examples} must be a list or NULL.") + } + + out <- list( + name = name, + version = version, + instructions = instructions, + schema = schema, + examples = examples, + created = foundry_utc_now() + ) + out$hash <- foundry_codebook_hash( + instructions = instructions, + schema = schema, + examples = examples, + version = version + ) + structure(out, class = "foundry_codebook") +} + + +#' Codebook schema helpers +#' +#' These light wrappers reuse foundryR's existing strict JSON Schema +#' constructors while following the measurement-layer codebook vocabulary. +#' +#' @param desc Character. Optional field description. +#' @param values Character vector of allowed values for `type_enum()`. +#' +#' @return A JSON Schema fragment represented as an R list. +#' @name codebook_schema_helpers +NULL + + +#' @rdname codebook_schema_helpers +#' @export +type_boolean <- function(desc = NULL) { + schema_boolean(description = desc) +} + + +#' @rdname codebook_schema_helpers +#' @export +type_enum <- function(desc = NULL, values) { + schema_enum(values = values, description = desc) +} + + +#' @rdname codebook_schema_helpers +#' @export +type_number <- function(desc = NULL) { + schema_number(description = desc) +} + + +#' @rdname codebook_schema_helpers +#' @export +type_string <- function(desc = NULL) { + schema_string(description = desc) +} + + +#' Compare two codebooks +#' +#' Print a compact diff of two `foundry_codebook` objects, including both +#' hashes, a unified diff of instructions, and field-level changes for schema +#' properties and examples. +#' +#' @param old,new `foundry_codebook` objects to compare. +#' +#' @return Invisibly returns the printed diff lines. +#' @export +#' +#' @examples +#' \dontrun{ +#' codebook_diff(old_codebook, new_codebook) +#' } +codebook_diff <- function(old, new) { + foundry_check_codebook(old, "old") + foundry_check_codebook(new, "new") + + lines <- c( + "Codebook diff", + paste0("old: ", old$name, " ", old$version, " ", old$hash), + paste0("new: ", new$name, " ", new$version, " ", new$hash), + "", + "Instructions:", + foundry_unified_diff(old$instructions, new$instructions), + "", + "Schema:", + foundry_named_list_diff(old$schema$properties, new$schema$properties), + "", + "Examples:", + foundry_named_list_diff( + foundry_examples_as_fields(old$examples), + foundry_examples_as_fields(new$examples) + ) + ) + + cat(lines, sep = "\n") + invisible(lines) +} + + +#' @export +format.foundry_codebook <- function(x, ...) { + foundry_check_codebook(x, "x") + + properties <- x$schema$properties %||% list() + variables <- if (length(properties) == 0L) { + " (none)" + } else { + vapply(names(properties), function(field) { + paste0(" - ", field, ": ", foundry_schema_field_summary(properties[[field]])) + }, character(1)) + } + + c( + paste0("foundry codebook: ", x$name), + paste0("version: ", x$version), + paste0("hash: ", substr(x$hash, 1L, 12L)), + "variables:", + variables, + paste0("examples: ", length(x$examples %||% list())) + ) +} + + +#' @export +print.foundry_codebook <- function(x, ...) { + cat(format(x), sep = "\n") + invisible(x) +} + + +foundry_codebook_hash <- function(instructions, schema, examples, version) { + payload <- list( + instructions = instructions, + schema = foundry_preserve_schema_arrays(schema), + examples = examples, + version = version + ) + json <- jsonlite::toJSON( + payload, + auto_unbox = TRUE, + digits = NA, + null = "null" + ) + digest::digest(enc2utf8(as.character(json)), algo = "sha256", serialize = FALSE) +} + + +foundry_check_codebook <- function(x, arg) { + if (!inherits(x, "foundry_codebook")) { + cli::cli_abort("{.arg {arg}} must be a {.cls foundry_codebook} object.") + } + invisible(x) +} + + +foundry_check_codebook_name <- function(name) { + foundry_check_character_scalar(name, "name") + if (!grepl("^[a-z0-9]+(-[a-z0-9]+)*$", name)) { + cli::cli_abort( + "{.arg name} must be a lowercase slug with optional hyphens." + ) + } + invisible(name) +} + + +foundry_check_semver <- function(version) { + foundry_check_character_scalar(version, "version") + semver <- paste0( + "^(0|[1-9][0-9]*)\\.", + "(0|[1-9][0-9]*)\\.", + "(0|[1-9][0-9]*)", + "(-[0-9A-Za-z.-]+)?", + "(\\+[0-9A-Za-z.-]+)?$" + ) + if (!grepl(semver, version)) { + cli::cli_abort("{.arg version} must be a semantic version string.") + } + invisible(version) +} + + +foundry_utc_now <- function() { + as.POSIXct(as.numeric(Sys.time()), origin = "1970-01-01", tz = "UTC") +} + + +foundry_unified_diff <- function(old, new) { + old_lines <- foundry_split_lines(old) + new_lines <- foundry_split_lines(new) + if (identical(old_lines, new_lines)) { + return(" (no changes)") + } + + c( + "--- old instructions", + "+++ new instructions", + "@@", + foundry_lcs_diff(old_lines, new_lines) + ) +} + + +foundry_split_lines <- function(x) { + if (identical(x, "")) { + return("") + } + strsplit(x, "\n", fixed = TRUE)[[1]] +} + + +foundry_lcs_diff <- function(old, new) { + n_old <- length(old) + n_new <- length(new) + lcs <- matrix(0L, nrow = n_old + 1L, ncol = n_new + 1L) + + if (n_old > 0L && n_new > 0L) { + for (i in seq_len(n_old)) { + for (j in seq_len(n_new)) { + if (identical(old[[i]], new[[j]])) { + lcs[i + 1L, j + 1L] <- lcs[i, j] + 1L + } else { + lcs[i + 1L, j + 1L] <- max(lcs[i, j + 1L], lcs[i + 1L, j]) + } + } + } + } + + foundry_lcs_backtrack(old, new, lcs, n_old, n_new) +} + + +foundry_lcs_backtrack <- function(old, new, lcs, i, j) { + out <- character() + while (i > 0L || j > 0L) { + if (i > 0L && j > 0L && identical(old[[i]], new[[j]])) { + out <- c(paste0(" ", old[[i]]), out) + i <- i - 1L + j <- j - 1L + } else if (j > 0L && (i == 0L || lcs[i + 1L, j] >= lcs[i, j + 1L])) { + out <- c(paste0("+", new[[j]]), out) + j <- j - 1L + } else { + out <- c(paste0("-", old[[i]]), out) + i <- i - 1L + } + } + out +} + + +foundry_named_list_diff <- function(old, new) { + old <- old %||% list() + new <- new %||% list() + + all_names <- union(names(old), names(new)) + if (length(all_names) == 0L) { + return(" (none)") + } + + lines <- vapply(all_names, function(field) { + old_has <- field %in% names(old) + new_has <- field %in% names(new) + if (!old_has) { + return(paste0("+ ", field, ": ", foundry_json_summary(new[[field]]))) + } + if (!new_has) { + return(paste0("- ", field, ": ", foundry_json_summary(old[[field]]))) + } + if (identical(foundry_canonical_json(old[[field]]), foundry_canonical_json(new[[field]]))) { + return(paste0(" ", field, ": no change")) + } + paste0( + "~ ", + field, + ": ", + foundry_json_summary(old[[field]]), + " -> ", + foundry_json_summary(new[[field]]) + ) + }, character(1)) + + if (all(grepl(": no change$", lines))) { + return(" (no changes)") + } + lines +} + + +foundry_examples_as_fields <- function(examples) { + if (is.null(examples)) { + return(list()) + } + if (is.null(names(examples)) || any(names(examples) == "")) { + names(examples) <- as.character(seq_along(examples)) + } + examples +} + + +foundry_json_summary <- function(x) { + json <- foundry_canonical_json(x) + if (nchar(json) > 80L) { + return(paste0(substr(json, 1L, 77L), "...")) + } + json +} + + +foundry_canonical_json <- function(x) { + as.character(jsonlite::toJSON( + foundry_preserve_schema_arrays(x), + auto_unbox = TRUE, + digits = NA, + null = "null" + )) +} + + +foundry_schema_field_summary <- function(field) { + type <- field$type %||% "unknown" + enum <- unclass(field$enum %||% NULL) + allowed <- if (is.null(enum)) { + "" + } else { + paste0(" [", paste(enum, collapse = ", "), "]") + } + description <- field$description %||% NULL + detail <- if (is.null(description)) "" else paste0(" (", description, ")") + paste0(type, allowed, detail) +} diff --git a/SPEC-measurement-layer.md b/SPEC-measurement-layer.md new file mode 100644 index 0000000..603e61d --- /dev/null +++ b/SPEC-measurement-layer.md @@ -0,0 +1,525 @@ +# foundryR Measurement Layer: Implementation Specification + +Status: draft for implementation +Owner: Alex (farach) +Audience: coding agent (Claude Code, Codex, Copilot) plus human reviewer +Repo: github.com/farach/foundryR + +--- + +## 0. How to use this spec (instructions to the coding agent) + +1. Read this entire file before writing any code. +1a. Enumerate the skills available in your environment (repo-level + `.claude/skills/`, user-level skills, or equivalent). If a skill router + skill exists, consult it FIRST and follow its routing for this task + before applying any other skill. Known relevant skill: `wti-dataviz`, + which is REQUIRED for any figures produced for the vignette or pkgdown + articles (M6). If no router or no skills are found, state that + explicitly in your plan and proceed using this spec alone. +1b. Scope discipline: every function, file, and exported symbol you create + must trace to a numbered section of this spec. If you believe code is + needed that has no spec anchor, STOP and propose a spec amendment; do + not write unanchored code. Unreferenced code found during review is + deleted, not justified after the fact. +2. Read the existing package source in `R/` to learn current conventions: + httr2 request construction, error handling style, tibble-first return + values, credential handling (`foundry_set_endpoint()`, `foundry_set_key()`, + `foundry_set_token()`), and the existing testthat setup. +3. Implement ONE milestone (Section 11) per pull request. Do not combine + milestones. Each PR must pass `devtools::check()` with no new errors, + warnings, or notes, and all new code must have tests. +4. Do not modify existing transport functions (`foundry_chat()`, + `foundry_response()`, `foundry_extract()`, `foundry_embed()`, batch and + file functions) except where a milestone explicitly says to add a + parameter. Reuse them internally. +5. If you encounter anything listed in Section 12 (Open questions reserved + for Alex), STOP and ask. Do not choose silently. +6. Before Milestone 5, fetch and read the current CRAN documentation for the + `ipd` package to verify its exact function signature and expected data + layout. Details of the ipd API in this spec are marked [Verify] and must + be confirmed against current docs, not assumed. + +--- + +## 1. Context and goal + +foundryR currently provides a tidy access layer to Microsoft Azure AI +Foundry. This spec adds a measurement layer that treats LLM annotation as a +research instrument: replicated annotation with full parameter capture, +reliability statistics, gold-set design and validation, a handoff to +inference-on-predicted-data estimators (the `ipd` package), and executable +provenance. + +Target user: a researcher or analyst who uses an LLM to label text at scale +and needs the resulting estimates to be defensible in a paper, report, or +regulatory context. + +Positioning sentence for docs and README: "Turn LLM output into estimates +you can defend." + +## 2. Non-goals + +- No new estimators. Bias correction is delegated to `ipd` (Suggests). +- No changes to image, video, or speech-synthesis functions. +- No provider abstraction in this phase. Foundry is the only backend, but + see Section 3 rule 4 for the seam that keeps extraction possible later. +- No Shiny or GUI components. +- No agent/MCP surface in this phase. +- No promotional, comms, or marketing content. pkgdown highlight articles + are governed by a separate brief owned by Alex; the only narrative + artifact in scope here is the flagship vignette (M6). Do not draft + announcement posts, taglines, or promotional copy under this spec. +- No multi-agent orchestration frameworks, swarm infrastructure, or + workflow engines. The only multi-agent structure permitted is the + sequential role protocol in Section 11 (implementer, reviewer, + CRAN skeptic), each run as an independent fresh-context session. + +## 3. Design principles + +1. Long format is canonical: one row per unit per replication. All + summaries (consensus, reliability) are derived views. +2. Every output row carries its own provenance columns. A row separated + from its tibble must still identify model, parameters, and codebook. +3. Idempotency: identical (text, codebook, model, params, rep) never pays + for tokens twice. Caching is content-addressed. +4. Transport seam: measurement functions must not call httr2 directly. + They call a single internal generic, `annotate_backend()`, whose default + method wraps the existing Foundry transport. This is the extraction seam + for a future provider-agnostic package. Keep it internal (not exported). +5. CRAN hygiene: hard dependencies limited to what the package already + imports plus `digest` and `jsonlite` (verify jsonlite is not already an + import). `ipd` goes in Suggests. Reliability math is implemented + natively, not via `irr` or similar. +6. Every user-facing message uses the package's existing messaging style + (cli if already used; otherwise match existing conventions). + +## 4. New files + +``` +R/codebook.R foundry_codebook(), foundry_schema(), codebook_diff(), + print/format methods, hashing +R/annotate.R foundry_annotate(), annotate_backend() internal generic, + realtime and batch drivers, budget guardrail +R/cache.R content-addressed cache: key construction, read/write, + foundry_cache_clear(), foundry_cache_status() +R/consensus.R foundry_consensus(), stability flags +R/reliability.R foundry_reliability(), native Cohen's kappa, + Krippendorff's alpha (nominal + ordinal), bootstrap CI, + confusion matrix, print and report methods +R/gold.R foundry_gold_design(), foundry_gold_join() +R/ipd.R foundry_to_ipd() adapter (+ optional thin wrapper) +R/provenance.R foundry_provenance(), methods_paragraph(), + write_provenance() +``` + +Tests mirror this layout under `tests/testthat/`. + +## 5. Data contracts + +### 5.1 Codebook object + +Class `foundry_codebook`. A list with elements: + +| element | type | notes | +|----------------|-----------|------------------------------------------------| +| name | chr(1) | slug, lowercase, hyphens allowed | +| version | chr(1) | semver string, validated | +| instructions | chr(1) | the system/instruction prompt | +| schema | list | JSON Schema list, built by foundry_schema() | +| examples | list|NULL | few-shot examples; part of the hash | +| created | POSIXct | UTC | +| hash | chr(1) | sha256 over canonical serialization of | +| | | (instructions, schema, examples, version) | + +Canonical serialization: `jsonlite::toJSON(..., auto_unbox = TRUE, +digits = NA, null = "null")` of a list with elements in the fixed order +above, then `digest::digest(algo = "sha256")` on the resulting string. +Document this so hashes are reproducible across sessions. + +`foundry_schema(...)` is a light constructor producing the JSON Schema list +in the same shape `foundry_extract()` already accepts. Provide helpers +`type_boolean(desc)`, `type_enum(desc, values)`, `type_string(desc)`, +`type_number(desc)`. If ellmer-style helpers already exist in the package, +reuse them instead of duplicating. + +`codebook_diff(old, new)` prints a unified diff of instructions and a +field-level diff of schema and examples, plus both hashes. + +Print method: renders like a content-analysis codebook (name, version, +hash prefix, variable list with types and allowed values, n examples). + +### 5.2 Annotation tibble + +Class `c("foundry_annotation", class(tibble::tibble()))`. + +Columns, in order (dot-prefixed to avoid collisions with user data): + +| column | type | notes | +|---------------------|---------|-------------------------------------------| +| {id_col} | as input| user's id column, name preserved | +| .text_hash | chr | sha256 of the unit text | +| .rep | int | 1..reps | +| {label columns} | varies | one column per schema property | +| .model | chr | model id requested | +| .deployment | chr | Foundry deployment name | +| .temperature | dbl | | +| .codebook | chr | codebook name | +| .codebook_version | chr | | +| .codebook_hash | chr | full sha256 | +| .response_id | chr | provider response id, NA if from cache | +| .prompt_tokens | int | | +| .completion_tokens | int | | +| .created | POSIXct | UTC | +| .from_cache | lgl | | + +Attributes: `codebook` (the codebook object), `run_meta` (list: endpoint +hash, api flavor, mode realtime/batch, seed, started/finished timestamps, +package version). Store the endpoint as a sha256 hash, never the raw URL, +so sidecars can be shared without leaking resource addresses. + +Failed units: on per-unit failure after retries, emit the row with NA +labels and add attribute `failures` (tibble of id, rep, error message). +Never silently drop rows; row count must equal n_units x reps. + +### 5.3 Provenance sidecar (JSON) + +Top-level keys (all snake_case): + +``` +provenance_version, created, +codebook {name, version, sha256, n_examples, schema}, +model {id, deployment, endpoint_sha256, api}, +parameters {temperature, reps, seed, mode}, +data {n_units, id_col, text_col, unit_hash_algo}, +gold {n, design, strata, seed} (null if absent), +usage {requests, cached, prompt_tokens, completion_tokens, est_cost_usd}, +reliability {alpha_reps, alpha_ci, kappa_vs_human, accuracy, per_class} + (null if absent), +estimate {method, formula, package_versions} (null if absent), +session {r_version, foundryR_version, os, locale} +``` + +`est_cost_usd` requires a pricing table; see Section 12 item 4 before +implementing (ship NULL if undecided). + +## 6. Function specifications + +### 6.1 foundry_annotate() + +```r +foundry_annotate( + data, text, codebook, + model = NULL, id = NULL, + reps = 1, temperature = 0.3, seed = NULL, + mode = c("realtime", "batch"), + budget_usd = NULL, max_requests = NULL, + cache = TRUE, verbose = TRUE, ... +) +``` + +- `text`, `id` use tidy evaluation (unquoted column names). If `id` is + NULL, create `.id` from row numbers and warn. +- Builds one request per (unit, rep) using the codebook instructions as + system prompt and the schema for structured output, reusing the internals + of `foundry_extract()` / `foundry_response()`. +- Before any network call: compute request count and estimated tokens, + print a plan line (n units, reps, requests, cache hits expected, mode). + If `budget_usd` or `max_requests` would be exceeded, abort with a clear + error before spending anything. In interactive sessions with more than + 1,000 non-cached requests and no explicit budget, ask for confirmation. +- Realtime mode: sequential or chunked requests through the existing + transport with retry/backoff matching package conventions. +- Batch mode: reuse `foundry_batch_requests()`, `foundry_file_upload()`, + `foundry_batch_create()`. Provide `foundry_annotate_status(job)` and + `foundry_annotate_collect(job)` so long jobs are resumable across + sessions; collect must merge results into the same annotation tibble + contract and write them into the cache. +- Cache behavior in Section 7. + +### 6.2 foundry_consensus() + +```r +foundry_consensus(ann, method = c("majority", "unanimous"), ties = "na") +``` + +Returns one row per unit: modal label per schema property, `.agreement` +(share of reps agreeing with mode), `.stable` (lgl, agreement >= threshold, +default 0.8), plus provenance columns collapsed where constant. Keeps class +`foundry_annotation` with attribute `collapsed = TRUE`. + +### 6.3 foundry_reliability() + +```r +foundry_reliability(ann, human = NULL, by = NULL, + level = c("nominal", "ordinal"), + boot = 1000, conf = 0.95, seed = NULL) +``` + +Computes, per label column (or the one named in `by`): + +1. Intra-model across reps: Krippendorff's alpha treating reps as coders; + percent unanimous; count of unstable units. +2. Model vs human (if `human` supplied, joined on the id column): + Cohen's kappa on consensus labels vs human, accuracy, sensitivity, + specificity (binary) or per-class precision/recall (multiclass), + confusion matrix. +3. Model vs model: if `ann` contains multiple `.model` values, pairwise + alpha between models on consensus labels. + +Bootstrap CIs for alpha: resample units with replacement, `boot` draws. + +Native implementations required, with unit tests against published worked +examples (Section 9). Return object class `foundry_reliability`: list of +tibbles plus metadata; `print()` renders the block format shown in the +package README example; `report(rel)` returns a character methods-style +paragraph fragment. + +### 6.4 foundry_gold_design() and foundry_gold_join() + +```r +foundry_gold_design(data, n, strata = NULL, seed = NULL) +``` + +Stratified (proportional) or simple random sampling. Returns a list of +class `foundry_gold_design`: `sample` (the selected rows with `.gold = +TRUE`), `record` (tibble: method, n, strata variable, seed, per-stratum +counts, created). The record is what provenance consumes; the seed is +mandatory in the record (generate and report one if the user passed NULL). + +```r +foundry_gold_join(ann, human, by = NULL, suffix = "_human") +``` + +Validates that human labels use the codebook's allowed values; errors on +unknown labels listing offenders. Joins onto the annotation tibble. + +### 6.5 foundry_to_ipd() + +```r +foundry_to_ipd(ann, human, outcome = NULL, collapse = "majority") +``` + +Produces the stacked data frame `ipd` expects: gold rows carry both the +human label (truth) and the model label (prediction); non-gold rows carry +the model label only; a `set_label` column distinguishes "labeled" and +"unlabeled". [Verify] Confirm required column naming and the `ipd::ipd()` +signature against current CRAN docs before implementing; adjust the +adapter, not the annotation contract, if they differ. Include a +`skip_on_cran`-guarded integration test that runs a small end-to-end +example when `ipd` is installed. + +### 6.6 foundry_provenance() and friends + +```r +foundry_provenance(ann, reliability = NULL, estimate = NULL) +methods_paragraph(prov, style = c("generic", "apa")) +write_provenance(prov, path) # writes path.json and path.md +``` + +`methods_paragraph()` is template-based (glue), not model-generated: +deterministic output, no network calls. It must gracefully omit sentences +for absent components (no reliability object, no estimate). `estimate` +handling: accept an `ipd` fit object if available and extract method name +and formula defensively; otherwise accept a plain list the user builds. + +## 7. Caching + +- Location: `tools::R_user_dir("foundryR", "cache")`, overridable via + option `foundryR.cache_dir`. +- Key: sha256 of the canonical JSON of (text, codebook_hash, model, + deployment, temperature, seed, rep_index, api_flavor). +- Value: the parsed label list plus usage metadata, stored as one JSON + file per key (no rds; keep it inspectable). +- `foundry_annotate(cache = TRUE)` reads before requesting and writes + after. `.from_cache` column records hits. +- `foundry_cache_status()` prints entry count and size; + `foundry_cache_clear(codebook = NULL)` clears all or one codebook hash. +- Never cache failures. + +## 8. Dependencies + +- Imports (new): `digest`; `jsonlite` if not already imported. +- Suggests (new): `ipd`. +- Do not add: `irr`, `boot`, `caret`, or any tidymodels component beyond + what the package already uses. + +## 9. Testing plan + +1. Transport mocking: use the package's existing HTTP mocking approach; if + none exists, introduce `httptest2` (Suggests) with recorded fixtures. + No test may hit the network on CRAN. +2. Reliability math against published values (hard requirement): + - Cohen's kappa: reproduce the worked example in Cohen (1960), and + cross-check one two-coder table against a hand-computed value in the + test comment. + - Krippendorff's alpha: reproduce the nominal-data worked example from + Krippendorff's published reliability notes (the widely reprinted + coder-pair example; document the exact source table in the test). + Tolerance 1e-6. + - Degenerate cases: single category (alpha undefined, return NA with + warning), all-agree, all-disagree, missing values. +3. Contract tests: column names, types, and row count invariants of the + annotation tibble, including the failure path (NA rows plus failures + attribute). +4. Cache tests: hit/miss, invalidation on codebook edit, corrupted cache + entry handled by re-request. +5. Snapshot tests: codebook print, reliability print, methods_paragraph + output, sidecar JSON (with volatile fields normalized). +6. Batch mode: unit-test request-file construction and collect/merge + using fixtures; do not test live batch jobs. + +## 10. Documentation + +- Roxygen for every exported function, with runnable `\dontrun{}` examples + matching this spec's usage. +- One flagship vignette: "From text to defensible estimates: LLM + annotation as measurement" following the job-tasks example (2,400 task + statements, binary ai_applicable, 5 reps, 250-unit stratified gold set, + PPI++ via ipd). Structure: codebook, gold design, annotate, reliability, + corrected estimate, provenance. Use a small bundled synthetic dataset so + the vignette builds without credentials; show real-call code in + non-evaluated chunks. +- README: add a "Measurement" section above the endpoint feature list and + move the positioning sentence to the first paragraph. +- pkgdown: new reference section "Measurement" listing these functions + first. + +## 11. Milestones (one PR each, in order) + +### 11.0 Verification loop (applies to EVERY milestone) + +Run this loop before declaring any milestone done. Do not skip steps. + +1. Implement against the milestone's spec sections only. +2. Self-review: re-read the relevant spec sections, then read your own + diff line by line and note every deviation. +3. Mechanical checks: run lintr (add a `.lintr` config in M1 if absent), + `devtools::check()` (zero new errors/warnings/notes), and the full + test suite. +4. Gap analysis: produce a table in the PR description listing every + requirement of the milestone (by spec section) with status MET, + PARTIAL, or NOT MET, with one line of evidence each (test name, file, + or line reference). PARTIAL or NOT MET items block the milestone. +5. Dead-code check: confirm no unexported, uncalled functions; no + commented-out blocks; no TODO markers without a linked issue; no new + exports beyond Sections 4 and 6. Delete anything unanchored per + Section 0 rule 1b. +6. Reviewer pass (fresh context): a NEW session/agent that did not write + the code reads only this spec and the diff, and independently repeats + steps 2, 4, and 5. The reviewer's findings are resolved before merge. + Implementer and reviewer must not share a context window. + +### 11.1 Sequential role protocol + +Three roles, always sequential, never concurrent, each a fresh session: +- Implementer: writes code for exactly one milestone. +- Reviewer: adversarial spec-conformance review (step 6 above). +- CRAN skeptic: runs once, at M7, per Section 11.9. +No other agent roles are authorized by this spec. + +M1 Codebook. codebook.R with constructor, schema helpers, hashing, print, + diff. Acceptance: hashes stable across sessions; snapshot tests pass; + check() clean. + +M2 Annotate (realtime) + cache. annotate.R, cache.R, the + annotate_backend() seam, budget guardrail, failure semantics. + Acceptance: contract tests pass against mocked transport; second + identical run performs zero network calls; exceeding budget aborts + before any request. + +M3 Annotate (batch). Batch driver reusing existing batch functions; + status/collect; cache writes on collect. Acceptance: fixture-based + round trip produces a contract-conformant tibble flagged mode=batch. + +M4 Consensus + reliability. consensus.R, reliability.R with native kappa + and alpha plus bootstrap CIs, print and report methods. Acceptance: + published-value tests pass at 1e-6; print snapshot matches spec format. + +M5 Gold + ipd adapter. gold.R, ipd.R. Acceptance: design record is + complete and seeded; label validation errors are informative; + ipd integration test (skipped on CRAN) runs end to end; [Verify] items + resolved against current ipd docs and noted in the PR description. + +M6 Provenance + vignette + README. provenance.R, sidecar writer, + methods_paragraph, flagship vignette, README and pkgdown updates. + Acceptance: sidecar validates against the schema in 5.3; vignette + builds without credentials; check() clean. All vignette and article + figures follow the wti-dataviz skill. + +### 11.9 M7 Red-team pre-mortem (after M6, before CRAN submission) + +Purpose: surface the strongest criticisms this package will face and +resolve each one BEFORE release. Run as three independent fresh-context +sessions, one per persona. Each persona reads the spec, the full package +source, the vignette, and pkgdown output, then writes its harshest +good-faith review. + +Personas: +- P1 rOpenSci-style package reviewer: API design, statistical software + standards, test coverage, documentation completeness, dependency + hygiene, CRAN policy compliance. +- P2 Methods-focused academic referee (quantitative social science / + economics): validity of the measurement claims, whether the vignette's + inferential workflow is defensible, whether limitations are honestly + stated. +- P3 Tidyverse-fluent R programmer: ergonomics, naming, tidy-eval + correctness, print methods, error messages, "why not just use ellmer + and mall" skepticism. + +Seed criticisms (each persona must address the relevant ones; finding +none of these applicable requires stated justification): +1. Reliability math is wrong or untested against authoritative values. +2. "LLM labels are not measurement": the package legitimizes treating + model output as data without sufficient methodological guardrails. +3. Reproducibility: results change across model versions and even + identical calls; seeds and caching give false comfort. +4. "This is just an API wrapper with extra steps." +5. Azure lock-in: the measurement layer is chained to one vendor. +6. Dependency and check hygiene: bloat, fragile Suggests behavior, + network calls in tests or vignettes. +7. Cost opacity: users cannot predict spend before committing. + +Resolution rule: every criticism raised must map to exactly one of +(a) a new or existing test, (b) a documentation or API change, or +(c) an explicit paragraph in a "Limitations" section of the flagship +vignette. Dismissal is not an available resolution. Output: a +`RED-TEAM.md` report at repo root listing each criticism, its +resolution type, and a link to the commit or file that resolves it, +plus GitHub issues for anything deferred with justification. +Acceptance: RED-TEAM.md complete; zero criticisms without a mapped +resolution; Alex signs off on every item resolved as type (c). + +## 12. Open questions reserved for Alex (agent must ask, not decide) + +1. Default temperature for replication (spec says 0.3; confirm) and + whether to expose a `models = c(...)` multi-model replication mode in + M2 or defer it. +2. Whether unstable units (low rep agreement) should be excluded by + default in foundry_to_ipd() or passed through with a flag. +3. Naming: `foundry_annotate` vs `foundry_code` (content-analysis + vocabulary). Spec assumes foundry_annotate. +4. Cost estimation table (per-model pricing) for est_cost_usd: include a + static table that can go stale, or ship NULL until a pricing endpoint + exists? +5. Whether methods_paragraph() should offer a citation string/BibTeX for + the package itself (relates to a future CITATION file and any paper). +6. Whether to submit for rOpenSci software review or JOSS after CRAN + acceptance. This affects M7 emphasis (rOpenSci standards checklists) + and the CITATION file. The agent prepares for it only if Alex says so. + +## 13. Style conventions + +- Match existing package style; tidyverse style guide otherwise. +- lintr must pass with the repo `.lintr` config; no per-line suppressions + without a comment explaining why. +- No dead code: no unexported uncalled functions, no commented-out + blocks, no TODOs without linked issues. +- Exported API surface is frozen to Sections 4 and 6 of this spec; any + additional export requires a spec amendment first. +- Errors and warnings must name the offending argument and show the + offending values (first 5, then a count). +- All timestamps UTC. All hashes sha256, lowercase hex. +- No new global state beyond the documented cache dir and options + prefixed `foundryR.`. +- NEWS.md entry per milestone. diff --git a/_pkgdown.yml b/_pkgdown.yml index c3edcb5..c016e72 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -151,6 +151,10 @@ reference: - schema_number - schema_integer - schema_boolean + - type_boolean + - type_enum + - type_number + - type_string - schema_array - schema_object - as_foundry_schema @@ -253,6 +257,8 @@ reference: contents: - foundry_agreement - foundry_consistency + - foundry_codebook + - codebook_diff - foundry_provenance - title: tidymodels integration diff --git a/man/codebook_diff.Rd b/man/codebook_diff.Rd new file mode 100644 index 0000000..3180ac7 --- /dev/null +++ b/man/codebook_diff.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/codebook.R +\name{codebook_diff} +\alias{codebook_diff} +\title{Compare two codebooks} +\usage{ +codebook_diff(old, new) +} +\arguments{ +\item{old, new}{\code{foundry_codebook} objects to compare.} +} +\value{ +Invisibly returns the printed diff lines. +} +\description{ +Print a compact diff of two \code{foundry_codebook} objects, including both +hashes, a unified diff of instructions, and field-level changes for schema +properties and examples. +} +\examples{ +\dontrun{ +codebook_diff(old_codebook, new_codebook) +} +} diff --git a/man/codebook_schema_helpers.Rd b/man/codebook_schema_helpers.Rd new file mode 100644 index 0000000..28ed925 --- /dev/null +++ b/man/codebook_schema_helpers.Rd @@ -0,0 +1,30 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/codebook.R +\name{codebook_schema_helpers} +\alias{codebook_schema_helpers} +\alias{type_boolean} +\alias{type_enum} +\alias{type_number} +\alias{type_string} +\title{Codebook schema helpers} +\usage{ +type_boolean(desc = NULL) + +type_enum(desc = NULL, values) + +type_number(desc = NULL) + +type_string(desc = NULL) +} +\arguments{ +\item{desc}{Character. Optional field description.} + +\item{values}{Character vector of allowed values for \code{type_enum()}.} +} +\value{ +A JSON Schema fragment represented as an R list. +} +\description{ +These light wrappers reuse foundryR's existing strict JSON Schema +constructors while following the measurement-layer codebook vocabulary. +} diff --git a/man/foundry_codebook.Rd b/man/foundry_codebook.Rd new file mode 100644 index 0000000..9deb02d --- /dev/null +++ b/man/foundry_codebook.Rd @@ -0,0 +1,50 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/codebook.R +\name{foundry_codebook} +\alias{foundry_codebook} +\title{Create a measurement codebook} +\usage{ +foundry_codebook(name, version, instructions, schema, examples = NULL) +} +\arguments{ +\item{name}{Character. Lowercase slug for the codebook; hyphens are allowed.} + +\item{version}{Character. Semantic version string.} + +\item{instructions}{Character. System or instruction prompt for annotation.} + +\item{schema}{List. JSON Schema object, typically from \code{\link[=foundry_schema]{foundry_schema()}}.} + +\item{examples}{List or \code{NULL}. Few-shot examples included in the codebook +hash.} +} +\value{ +A \code{foundry_codebook} object. +} +\description{ +A codebook records the instructions, JSON Schema, examples, semantic +version, creation time, and deterministic SHA-256 hash for an LLM annotation +instrument. The hash is computed from a canonical JSON serialization of +\code{instructions}, \code{schema}, \code{examples}, and \code{version}, in that order. Before +serialization, schema arrays are preserved with the same internal helper used +by structured outputs so single-value \code{enum} and \code{required} arrays do not +collapse to scalars. The payload is serialized with +\code{jsonlite::toJSON(auto_unbox = TRUE, digits = NA, null = "null")}, +normalized with \code{enc2utf8()}, and hashed with SHA-256. +} +\examples{ +\dontrun{ +codebook <- foundry_codebook( + name = "ai-applicability", + version = "1.0.0", + instructions = "Label whether the task could use AI assistance.", + schema = foundry_schema( + ai_applicable = type_boolean("AI could materially assist the task") + ), + examples = list( + list(text = "Draft a memo", ai_applicable = TRUE), + list(text = "Lift a heavy box", ai_applicable = FALSE) + ) +) +} +} diff --git a/tests/testthat/_snaps/codebook.md b/tests/testthat/_snaps/codebook.md new file mode 100644 index 0000000..650a018 --- /dev/null +++ b/tests/testthat/_snaps/codebook.md @@ -0,0 +1,73 @@ +# foundry_codebook validates name, version, schema, and examples + + Code + foundry_codebook(name = "Bad Name", version = "1.0.0", instructions = "Label.", + schema = foundry_schema(label = type_string())) + Condition + Error in `foundry_check_codebook_name()`: + ! `name` must be a lowercase slug with optional hyphens. + +--- + + Code + foundry_codebook(name = "good-name", version = "1", instructions = "Label.", + schema = foundry_schema(label = type_string())) + Condition + Error in `foundry_check_semver()`: + ! `version` must be a semantic version string. + +--- + + Code + foundry_codebook(name = "good-name", version = "1.0.0", instructions = "Label.", + schema = list(type = "string")) + Condition + Error in `as_foundry_schema()`: + ! `x` is not a supported schema object. + +--- + + Code + foundry_codebook(name = "good-name", version = "1.0.0", instructions = "Label.", + schema = foundry_schema(label = type_string()), examples = "not-list") + Condition + Error in `foundry_codebook()`: + ! `examples` must be a list or NULL. + +# codebook print and diff output are stable + + Code + print(old) + Output + foundry codebook: task-label + version: 1.0.0 + hash: a74264284ddf + variables: + - label: string [yes, no] (Task label) + examples: 1 + +--- + + Code + codebook_diff(old, new) + Output + Codebook diff + old: task-label 1.0.0 a74264284ddf870634ed8924c5b5e11fba0c53829792b220195f70811929fb2a + new: task-label 1.1.0 0a6994011e7b981609194e7fe0e6b681908faf0f76c7083212052ed8aa79afe2 + + Instructions: + --- old instructions + +++ new instructions + @@ + Label each task. + -Use yes or no. + +Use yes, no, or maybe. + + Schema: + ~ label: {"type":"string","description":"Task label","enum":["yes","no"]} -> {"type":"string","description":"Task label","enum":["yes","no","maybe"]} + + rationale: {"type":"string","description":"Short reason"} + + Examples: + 1: no change + + 2: {"text":"Carry a box","label":"no"} + diff --git a/tests/testthat/test-codebook.R b/tests/testthat/test-codebook.R new file mode 100644 index 0000000..7d8b6d0 --- /dev/null +++ b/tests/testthat/test-codebook.R @@ -0,0 +1,174 @@ +test_that("foundry_codebook builds the specified object contract", { + schema <- foundry_schema( + ai_applicable = type_boolean("AI could materially assist the task"), + confidence = type_number("Coder confidence") + ) + + codebook <- foundry_codebook( + name = "ai-applicability", + version = "1.0.0", + instructions = "Label whether the task could use AI assistance.", + schema = schema, + examples = list(list(text = "Draft a memo", ai_applicable = TRUE)) + ) + + expect_s3_class(codebook, "foundry_codebook") + expect_named( + codebook, + c("name", "version", "instructions", "schema", "examples", "created", "hash") + ) + expect_equal(codebook$name, "ai-applicability") + expect_equal(codebook$version, "1.0.0") + expect_equal(codebook$schema, schema) + expect_s3_class(codebook$created, "POSIXct") + expect_equal(attr(codebook$created, "tzone"), "UTC") + expect_match(codebook$hash, "^[0-9a-f]{64}$") +}) + +test_that("codebook hashes are stable and content addressed", { + schema <- foundry_schema(label = type_enum("Task label", c("yes", "no"))) + single_enum_schema <- foundry_schema(label = type_enum("Only label", "yes")) + first <- foundry_codebook( + name = "task-label", + version = "1.0.0", + instructions = "Label each task.", + schema = schema, + examples = list(list(text = "Write code", label = "yes")) + ) + second <- foundry_codebook( + name = "renamed-task-label", + version = "1.0.0", + instructions = "Label each task.", + schema = schema, + examples = list(list(text = "Write code", label = "yes")) + ) + changed <- foundry_codebook( + name = "task-label", + version = "1.0.1", + instructions = "Label each task.", + schema = schema, + examples = list(list(text = "Write code", label = "yes")) + ) + single_enum <- foundry_codebook( + name = "single-label", + version = "1.0.0", + instructions = "Label each task.", + schema = single_enum_schema, + examples = NULL + ) + + canonical <- jsonlite::toJSON( + list( + instructions = "Label each task.", + schema = foundry_preserve_schema_arrays(schema), + examples = list(list(text = "Write code", label = "yes")), + version = "1.0.0" + ), + auto_unbox = TRUE, + digits = NA, + null = "null" + ) + single_enum_canonical <- jsonlite::toJSON( + list( + instructions = "Label each task.", + schema = foundry_preserve_schema_arrays(single_enum_schema), + examples = NULL, + version = "1.0.0" + ), + auto_unbox = TRUE, + digits = NA, + null = "null" + ) + + expect_equal(first$hash, second$hash) + expect_equal( + first$hash, + digest::digest(enc2utf8(as.character(canonical)), algo = "sha256", serialize = FALSE) + ) + expect_equal( + single_enum$hash, + digest::digest( + enc2utf8(as.character(single_enum_canonical)), + algo = "sha256", + serialize = FALSE + ) + ) + expect_match(changed$hash, "^[0-9a-f]{64}$") + expect_failure(expect_equal(first$hash, changed$hash)) +}) + +test_that("codebook schema helpers reuse existing schema constructors", { + expect_equal(type_boolean("Flag"), schema_boolean("Flag")) + expect_equal(type_number("Score"), schema_number("Score")) + expect_equal(type_string("Text"), schema_string("Text")) + expect_equal( + type_enum("Choice", c("yes", "no")), + schema_enum(c("yes", "no"), description = "Choice") + ) +}) + +test_that("foundry_codebook validates name, version, schema, and examples", { + expect_snapshot( + error = TRUE, + foundry_codebook( + name = "Bad Name", + version = "1.0.0", + instructions = "Label.", + schema = foundry_schema(label = type_string()) + ) + ) + expect_snapshot( + error = TRUE, + foundry_codebook( + name = "good-name", + version = "1", + instructions = "Label.", + schema = foundry_schema(label = type_string()) + ) + ) + expect_snapshot( + error = TRUE, + foundry_codebook( + name = "good-name", + version = "1.0.0", + instructions = "Label.", + schema = list(type = "string") + ) + ) + expect_snapshot( + error = TRUE, + foundry_codebook( + name = "good-name", + version = "1.0.0", + instructions = "Label.", + schema = foundry_schema(label = type_string()), + examples = "not-list" + ) + ) +}) + +test_that("codebook print and diff output are stable", { + old <- foundry_codebook( + name = "task-label", + version = "1.0.0", + instructions = "Label each task.\nUse yes or no.", + schema = foundry_schema(label = type_enum("Task label", c("yes", "no"))), + examples = list(list(text = "Write code", label = "yes")) + ) + new <- foundry_codebook( + name = "task-label", + version = "1.1.0", + instructions = "Label each task.\nUse yes, no, or maybe.", + schema = foundry_schema( + label = type_enum("Task label", c("yes", "no", "maybe")), + rationale = type_string("Short reason") + ), + examples = list( + list(text = "Write code", label = "yes"), + list(text = "Carry a box", label = "no") + ) + ) + + expect_snapshot(print(old)) + expect_snapshot(codebook_diff(old, new)) +})