From 73f9e783a6fbd4e03d03d718cd9ef4160134f8f4 Mon Sep 17 00:00:00 2001 From: Scotty Pate Date: Tue, 11 Aug 2026 06:36:48 -0500 Subject: [PATCH] Context v2: name the binding, show the measurement, stop the false URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a design partner consuming a live bound cell. Four document changes, one breaking, plus the OpenAPI surface that let two of them go unnoticed. - declared.exports[].binding {source, object, connection}, present iff `query` is null. A bound export's target existed only in docs prose and `data.channels`' operator hints, so writing a query against a cell that serves no rows meant reading English. Values are verbatim cell.yaml, never profile-resolved: `table` is env-expandable and `Declared` is hashed into interface_digest, so a resolved value would fold the environment into the digest. - observed.source_check.exports[route] {check, grain, rows, distinct_grain}. verify::grain_counts computed the numbers behind `grain_verified`, compared them, and discarded them. Persisted through .cell/source_check.json under the record's existing digest+profile gate, visibility-filtered on the way to the wire. A grainless export contributes no entry — no check ran on it. - MeshExport.bound, so an agent routing off the manifest stops picking an export that 404s. The emitter accepts datamk_context 1 or 2. - declared.docs[].path -> source_path, which is why datamk_context is 2. A cell.yaml-relative path named `path` in a JSON document served over HTTP reads as a relative URL and 404s — the same false affordance already corrected for `route`. /context's 200 had no schema at all, so nothing said include= content lands at a top-level key named for the section rather than inside `included` (which holds section names). It has one now, pinned to ContextDocument's keys by a test. Data path items carry x-datamk-version and x-datamk-contract, previously prose inside `summary`. Declined, with reasoning in the ADR: a cell-level semver (no cell version exists; it would drift from the per-export semvers) and a structured `caveats` array (free-text rule + unenforced severity is prose in a JSON wrapper, drifting like source_descriptions already does). The hazard caveats aimed at — two USD columns that must not be subtracted — stays open; the only fix that isn't docs-in-JSON is a typed column property the engine can verify. Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/0012-cell-context-document.md | 81 +++++++- docs/adr/0013-long-form-docs-pages.md | 2 +- docs/guides/context.md | 50 ++++- src/context.rs | 181 ++++++++++++++++- src/engine/mod.rs | 5 +- src/manifest.rs | 17 ++ src/mesh.rs | 36 +++- src/serve/mod.rs | 11 +- src/serve/openapi.rs | 269 ++++++++++++++++++++++++- src/verify.rs | 32 ++- test/integrations/cli.rs | 2 +- 11 files changed, 654 insertions(+), 32 deletions(-) diff --git a/docs/adr/0012-cell-context-document.md b/docs/adr/0012-cell-context-document.md index 9ab0a04..7cbede9 100644 --- a/docs/adr/0012-cell-context-document.md +++ b/docs/adr/0012-cell-context-document.md @@ -309,7 +309,7 @@ is a registry endpoint and pre-auth crawl bait. ```json { "datamk_mesh": 1, "generated_at": "…", "cells": [ { "name": "…", "url": "…", "description": "…", - "exports": [ { "name": "…", "version": "…", "contract": "…" } ], + "exports": [ { "name": "…", "version": "…", "contract": "…", "bound": false } ], "context_digest": "…", "auth_hint": "…" } ] } ``` @@ -654,3 +654,82 @@ change): `never_backed_routes` -> `bound_routes`, `note_never_backed` -> `describe_never_offender` keep their names — they name the literal rejected `materialize: never` keyword the migration error matches on, which is correct as long as that error exists. + +## Amendment (2026-08-11): `datamk_context: 2` — bindings, measurements, and a rename + +From a design partner's report of consuming a live bound cell. Four changes, +one of them breaking. + +- **`declared.exports[].binding`** (additive). A bound export's target was + machine-invisible: `query: null` said "not here" and nothing said where. + The object name existed only in `docs:` prose, and `data.channels` — an + operator hint from the profile, free-form by design — gave the dataset at + best. Writing a query against a bound cell meant reading English, which is + the one thing this document exists to stop. Now: `{source, object, + connection}`, present iff `query` is null, so the two are complements. + **Values are verbatim `cell.yaml`, never profile-resolved** — `table` is + env-expandable and `Declared` is hashed wholesale into `interface_digest`, + so a resolved value would fold the environment into the digest and churn it + per deployment; a templated table ships as `${DATASET}.fct_x`. A `cell:` + source is rejected at resolve time (`verify::validate_bound_exports`) and an + unresolvable name discloses no object, so neither can leak an upstream's + table through this field — the §5 disclosure boundary is unchanged. + `materialize: "never"` was requested as a third key and declined: that + strategy was removed by the amendment above, and presence of `binding` is + already the positive assertion. +- **`observed.source_check.exports`** (additive). `grain_verified: true` and + `outcome: "passed"` asserted a result while `verify` computed the counts + behind them, compared them, and discarded them (`verify::grain_counts`). + Route key -> `{check, grain, rows, distinct_grain}`, persisted through + `.cell/source_check.json` under the same fail-closed digest+profile gate as + the record itself, and visibility-filtered on the way to the wire. + Timestamped by the enclosing `checked_at`: one pass, one time. A grainless + export contributes no entry — no check ran on it, and §2 forbids inventing + one. Deliberately not hung off `observed.exports` (`ExportProbe`), which is + lake-row-derived at swap time and would be re-meaned. +- **`MeshExport.bound`** (additive, mesh manifest). The manifest carried + `{name, version, contract}`, so an agent routing off it picked a bound + export and hit the 404 the document could have warned it about. Copied from + `binding`'s presence, like every other manifest field. The emitter now + accepts `datamk_context` 1 or 2 — the v2 rename touches nothing it copies. +- **`declared.docs[].path` -> `source_path`** (**breaking**; the reason + `datamk_context` is 2). A cell.yaml-relative filesystem path named `path`, + in a JSON document served over HTTP, reads as a relative URL and 404s for + anyone who tries it — the same false affordance already corrected for + `route` in the amendment above, in the same document, on the same reasoning. + Serving the pages at their own route was rejected again: §4 is one document, + one route, and `?include=docs` already delivers content. + +Alongside, on the OpenAPI surface (no document change): `/context`'s `200` +carried a bare description string and no schema at all, so nothing in the +spec said `include=`'s content lands at a **top-level key named for the +section** rather than inside `included` — the reported failure was an +iterator written over `included`, which holds section names. The response now +has a real schema, pinned to `ContextDocument`'s top-level keys by a test, +and the `include` parameter states the landing rule. Data path items also +carry `x-datamk-version` and `x-datamk-contract`, which previously existed +only as prose inside `summary`. + +**Declined: a cell-level semver alongside `info.version`'s digest.** There is +no cell version in `CellDef`, and adding one creates a second identity axis +that will drift from the per-export semvers within a single release, with no +policy for who bumps it. Exports version independently — that is the point of +the interface. `info.version` = interface digest remains correct (OpenAPI +attaches no semantics to that field); the real gap was that `contract` was +unreachable from `openapi.json`, which the extensions above close. Reverses +if consumers need to pin the cell as a whole — and the answer then is a +manifest of export versions, still not a new semver. + +**Declined: a structured `caveats` array per export, and docs-by-default.** +The motivating hazard is real — `mrr` and `total_infra_cost_usd` both +`decimal`/`USD`, with "not period-aligned" living only in prose behind an +optional flag, so subtraction looks fine to a JSON-only agent. But a +free-text `rule` with an engine-unenforced `severity` is prose in a JSON +wrapper: it drifts exactly like `observed.source_descriptions` already does, +and nothing can check it. Docs-by-default was rejected separately — it breaks +the ETag variant split (ADR 0013 §6), unbounds the default response, and +prose is precisely what the agent in question won't parse. The hazard stays +open. The only fix that isn't docs-in-JSON is a typed column property the +engine can verify — a period/point-in-time attribute beside `unit`, making +"these two are not comparable" derivable rather than asserted. Not decided +here. diff --git a/docs/adr/0013-long-form-docs-pages.md b/docs/adr/0013-long-form-docs-pages.md index e26c1aa..2a79c6a 100644 --- a/docs/adr/0013-long-form-docs-pages.md +++ b/docs/adr/0013-long-form-docs-pages.md @@ -153,7 +153,7 @@ key" impossible to reject on purpose. ## 5. Document shape — three places, never collapsed -- **`declared.docs`** — identity only: `{target, path, media_type}` per +- **`declared.docs`** — identity only: `{target, source_path, media_type}` per entry. `target` is `"cell"` or the route key (`name@major`) — route keys always carry `@major`, so no collision with the literal string `"cell"`. Always present (`[]` when none), alongside an affordance field diff --git a/docs/guides/context.md b/docs/guides/context.md index 7f7af58..62f087a 100644 --- a/docs/guides/context.md +++ b/docs/guides/context.md @@ -64,7 +64,7 @@ omitted or `null` — never fabricated, never zeros. ```json { - "datamk_context": 1, + "datamk_context": 2, "cell": "orders", "status": "verified", "grain_verified": true, @@ -258,10 +258,17 @@ curl -H "Authorization: Bearer $TOKEN" \ } ``` +`included` holds section *names*, never the content itself: each name in it +appears as a **top-level key of the same name**, keyed by +`declared.docs[].target`. Iterating `included` expecting objects gets you +strings. + The default `GET /context` (no `include`) never carries page content — only -`declared.docs`, the identity of every declared page (`{target, path, +`declared.docs`, the identity of every declared page (`{target, source_path, media_type}`, no bytes) and `declared.include_request`, the affordance -telling an agent how to ask for the rest. `included` is always present +telling an agent how to ask for the rest. `source_path` is the author's +path on disk, not a URL — it is deliberately not fetchable, and its name +says so. `included` is always present (`[]` on the default document, `["docs"]` once inlined) so an agent can tell "this server predates docs pages" (the field is absent) from "this cell just has none" (present, `docs` is `{}`). `?include=docs` on a @@ -369,6 +376,28 @@ refuses outright and points at `verify`/`context` instead: the contract is still real, but the Builder isn't the workload that proves it — a live check is. +The document says so in machine-readable form. A bound export carries +`query: null` — there is no `GET /{route}` for it, ever — and a `binding` +block naming where the rows actually are: + +```json +"exports": [{ + "name": "customer_pii", + "route": "customer_pii@1", + "query": null, + "binding": { "source": "pii", "object": "raw.customers", "connection": "crm" } +}] +``` + +The two are complements: exactly one of them is present on every export, so +"can I query this here, and if not where do I go" is one field lookup, not a +prose read. `object` and `connection` are verbatim `cell.yaml` — never +profile-resolved, so a templated table ships as `${DATASET}.raw_customers` +and the same document is honest in every environment. Which project or +account `crm` resolves to stays in the profile, as it always has. +`data.channels` is unchanged and still complementary: free-form operator +hints about the destination, where `binding` is the object itself. + `datamk verify` proves it: it binds the cell's sources against the live warehouse, then runs the exact same schema and grain checks it always has — declared columns exist with compatible types, declared grain exists and is @@ -428,12 +457,25 @@ as of the moment it ran, which may since have changed. An agent that reads "source_check": { "outcome": "passed", "checked_at": "2026-08-07T10:00:00Z", - "datamk_version": "0.0.14" + "datamk_version": "0.0.14", + "exports": { + "customer_pii@1": { + "check": "grain_unique", + "grain": ["id"], + "rows": 722, + "distinct_grain": 722 + } + } } } } ``` +`exports` carries what each check actually measured, so a reader sees what +passed and not merely that it did — `checked_at` timestamps all of them, one +pass, one time. An export with no declared grain contributes no entry: no +check ran on it, and an empty measurement is never invented to fill the gap. + `data_as_of` joins that block only when a connector can say, cheaply and truthfully, when the checked rows were last known-true — omitted otherwise, never guessed and never defaulted to `checked_at`. diff --git a/src/context.rs b/src/context.rs index f393a9b..3872d84 100644 --- a/src/context.rs +++ b/src/context.rs @@ -22,7 +22,9 @@ use crate::engine::run_summary::RunSummary; /// cell semver and `datamk_version`: additive changes don't bump; any removal, /// rename, or re-meaning bumps, with the prior version served through a /// deprecation window (ADR 0012 §2). -pub const DATAMK_CONTEXT_VERSION: u32 = 1; +/// +/// **2**: `declared.docs[].path` renamed to `source_path`. +pub const DATAMK_CONTEXT_VERSION: u32 = 2; /// The `limit` in every emitted `sample_request` — the smallest useful legal /// call, a pure function of the route key and the limit grammar. @@ -141,7 +143,11 @@ pub struct DeclaredDocsEntry { /// `"cell"` or the route key (`name@major`) — route keys always carry /// `@major`, so an export can never collide with the literal `"cell"`. pub target: String, - pub path: String, + /// The author's cell.yaml-relative filesystem path. Named `source_path`, + /// not `path`, because it is not fetchable — there is no `/docs/:target` + /// route (ADR 0012 §4); content arrives via `?include=docs`. Same + /// false-affordance fix already made for `DeclaredExport::route`. + pub source_path: String, pub media_type: String, } @@ -198,6 +204,25 @@ pub struct DeclaredExport { /// that do not exist there. #[serde(skip_serializing_if = "Option::is_none")] pub query: Option, + /// Where the rows are, for a bound export — present iff `query` is null. + #[serde(skip_serializing_if = "Option::is_none")] + pub binding: Option, +} + +/// A bound export's target, exactly as `cell.yaml` writes it — never +/// profile-resolved: `table` is env-expandable and `declared` is hashed into +/// `interface_digest`, so a resolved value would churn the digest per +/// environment. A templated table ships as `${DATASET}.fct_x`. +#[derive(Debug, Clone, Serialize)] +pub struct BindingBlock { + /// The `sources:` key this export binds to. + pub source: String, + /// The declared object: a warehouse table path, or a raw file/glob. + #[serde(skip_serializing_if = "Option::is_none")] + pub object: Option, + /// The connection alias only — what it resolves to is profile. + #[serde(skip_serializing_if = "Option::is_none")] + pub connection: Option, } /// One declared column as the document emits it. @@ -394,15 +419,51 @@ pub struct SourceCheck { #[serde(skip_serializing_if = "Option::is_none")] pub data_as_of: Option, pub datamk_version: String, + /// What each check actually measured, per route — the numbers behind + /// `outcome`, so a reader sees what passed and not only that it did. + /// Timestamped by `checked_at` above: one pass, one time. + #[serde(skip_serializing_if = "IndexMap::is_empty")] + pub exports: IndexMap, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ExportCheck { + pub check: String, + pub grain: Vec, + pub rows: i64, + pub distinct_grain: i64, } -impl From<&crate::manifest::SourceCheckRecord> for SourceCheck { - fn from(r: &crate::manifest::SourceCheckRecord) -> Self { +impl SourceCheck { + /// Visibility-filtered (ADR 0012 §4): a private export's measurement never + /// reaches the wire, so this takes the same route list every other + /// consumer reads rather than copying the record's map wholesale. + pub fn from_record( + r: &crate::manifest::SourceCheckRecord, + routes: &[(String, Export)], + ) -> Self { + let exports = routes + .iter() + .filter_map(|(route, _)| { + r.exports.get(route).map(|m| { + ( + route.clone(), + ExportCheck { + check: m.check.clone(), + grain: m.grain.clone(), + rows: m.rows, + distinct_grain: m.distinct_grain, + }, + ) + }) + }) + .collect(); SourceCheck { outcome: r.outcome.clone(), checked_at: r.checked_at.clone(), data_as_of: r.data_as_of.clone(), datamk_version: r.datamk_version.clone(), + exports, } } } @@ -504,6 +565,7 @@ pub fn declared(def: &CellDef, routes: &[(String, Export)]) -> Declared { .map(|(col, spec)| (col.clone(), ColumnDoc::from(spec))) .collect(), query: (!e.is_bound()).then(|| query_block(route, e)), + binding: e.bind.as_deref().map(|b| binding_block(def, b)), }) .collect(); @@ -530,6 +592,25 @@ pub fn declared(def: &CellDef, routes: &[(String, Export)]) -> Declared { } } +/// A bound export's target, looked up in `sources:`. A `cell:` source is +/// rejected at resolve time (`verify::validate_bound_exports`) and an unknown +/// name can't resolve, so both emit the source name alone rather than a +/// fabricated object. +fn binding_block(def: &CellDef, bind: &str) -> BindingBlock { + let (object, connection) = match def.sources.get(bind) { + Some(Source::Raw(path)) => (Some(path.clone()), None), + Some(Source::Connection { + connection, table, .. + }) => (table.clone(), Some(connection.clone())), + Some(Source::Cell { .. }) | None => (None, None), + }; + BindingBlock { + source: bind.to_string(), + object, + connection, + } +} + /// Docs identity only (ADR 0013): the cell-level page (if declared) plus /// every **discoverable** export's page, in that order — no filesystem /// access, since identity needs only the declared path and its extension. A @@ -540,7 +621,7 @@ fn docs_entries(def: &CellDef, routes: &[(String, Export)]) -> Vec Vec what the grain check actually measured. `#[serde(default)]` + /// so records written before this field still parse (as empty — no + /// measurement, never a fabricated one). + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub exports: BTreeMap, +} + +/// The numbers behind `grain_verified` for one export: the 722/722 that used +/// to be computed, compared, and thrown away. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GrainMeasurement { + /// The check that ran. `"grain_unique"` today; a closed vocabulary. + pub check: String, + /// The grain columns it ran on. + pub grain: Vec, + pub rows: i64, + pub distinct_grain: i64, } impl SourceCheckRecord { diff --git a/src/mesh.rs b/src/mesh.rs index 0ec5f3f..d09d327 100644 --- a/src/mesh.rs +++ b/src/mesh.rs @@ -57,6 +57,11 @@ pub struct MeshExport { pub name: String, pub version: String, pub contract: String, + /// Copied from the document: this export serves no rows over HTTP. Without + /// it an agent routing off the manifest picks a bound export and gets a + /// 404 with no warning it could have read here. + #[serde(default)] + pub bound: bool, } /// The hand-authored cells file (`mesh emit --cells`): the `{name, url}` @@ -233,7 +238,12 @@ fn summarize(entry: CellEntry, fetched: Option<(serde_json::Value, Option `source_path`) + // touches nothing the emitter copies. Unknown versions still bail. + if !matches!( + doc.get("datamk_context").and_then(|v| v.as_u64()), + Some(1) | Some(2) + ) { tracing::warn!(cell = %cell.name, "unrecognized datamk_context version; emitting name+url only"); return cell; } @@ -246,6 +256,9 @@ fn summarize(entry: CellEntry, fetched: Option<(serde_json::Value, Option> = @@ -464,12 +464,17 @@ fn observed_bundle_sha12( return None; } let checked_at = source_check.map(|sc| sc.checked_at.as_str()).unwrap_or(""); + // The measurements ride the same variant: two checks a second apart with + // different counts must not share an ETag. + let measurements_json = source_check + .map(|sc| serde_json::to_string(&sc.exports).unwrap_or_default()) + .unwrap_or_default(); // `source_descriptions` is deterministic within one process's lifetime // (built once, at startup, from a `BTreeMap`-sorted record) — good // enough for a cache-invalidation hash, which only needs to change when // the content genuinely does, not to be canonical across processes. let descriptions_json = serde_json::to_string(source_descriptions).unwrap_or_default(); - let joined = format!("{checked_at}|{descriptions_json}"); + let joined = format!("{checked_at}|{measurements_json}|{descriptions_json}"); Some(crate::context::sha256_hex(joined.as_bytes())[..12].to_string()) } @@ -1988,7 +1993,7 @@ mod smoke { let (status, body) = get(&router, "/context", None).await; assert_eq!(status, StatusCode::OK, "{body}"); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); - assert_eq!(v["datamk_context"], 1); + assert_eq!(v["datamk_context"], 2); assert_eq!(v["cell"], "smoke"); assert_eq!(v["status"], "draft"); assert_eq!(v["grain_verified"], false); diff --git a/src/serve/openapi.rs b/src/serve/openapi.rs index 466b6de..27584c0 100644 --- a/src/serve/openapi.rs +++ b/src/serve/openapi.rs @@ -71,11 +71,19 @@ fn context_path_item() -> Value { "style": "form", "explode": false, "description": "Comma-separated optional sections to inline. Omit for the \ - default document; `docs` inlines every declared docs page.", + default document; `docs` inlines every declared docs page. \ + Each section named in the response's `included` array is \ + inlined at a TOP-LEVEL key of the same name — `include=docs` \ + echoes `\"included\": [\"docs\"]` and puts the content in \ + `docs`, keyed by `declared.docs[].target`. `included` holds \ + section names, never the content itself.", "schema": { "type": "array", "items": { "type": "string", "enum": sections } } }], "responses": { - "200": { "description": "the context document" }, + "200": { + "description": "the context document", + "content": { "application/json": { "schema": context_schema() } } + }, "304": { "description": "not modified (If-None-Match matched the current ETag \ for the requested variant)" }, "400": { "description": "unknown query parameter, or an unrecognized/empty \ @@ -88,6 +96,192 @@ fn context_path_item() -> Value { }) } +/// The context document's response shape. Hand-written against +/// `context::ContextDocument` and pinned to it by +/// `context_schema_names_every_top_level_document_key` — the `/context` 200 +/// used to be a bare description string, so nothing in the spec said where +/// `include=` content lands or that `binding` exists. +fn context_schema() -> Value { + json!({ + "type": "object", + "required": ["datamk_context", "cell", "status", "grain_verified", + "declared", "observed", "data", "notes", "included"], + "properties": { + "datamk_context": { + "type": "integer", + "const": crate::context::DATAMK_CONTEXT_VERSION, + "description": "Document-schema version." + }, + "cell": { "type": "string" }, + "status": { + "type": "string", + "enum": ["draft", "verified_at_source", "verified"], + "description": "Weakest to strongest. `verified` means a published, \ + verify-gated execution stands behind this document." + }, + "grain_verified": { "type": "boolean" }, + "declared": { + "type": "object", + "description": "Author claims. Never flattened with `observed`.", + "required": ["exports", "upstreams", "docs", "include_request"], + "properties": { + "description": { "type": "string" }, + "exports": { "type": "array", "items": declared_export_schema() }, + "upstreams": { "type": "array", "items": { "type": "object", "properties": { + "ref": { "type": "string" }, + "version": { "type": "integer" } + }}}, + "docs": { "type": "array", "items": { "type": "object", + "required": ["target", "source_path", "media_type"], + "properties": { + "target": { "type": "string", + "description": "`cell`, or an export's route key." }, + "source_path": { "type": "string", + "description": "The author's cell.yaml-relative file path. \ + NOT a URL and not fetchable — there is no \ + /docs route; use ?include=docs for content." }, + "media_type": { "type": "string" } + }}}, + "include_request": { "type": "string" } + } + }, + "observed": { + "type": ["object", "null"], + "description": "Machine facts, or null when nothing has been built or \ + verified. Absent facts are omitted, never fabricated.", + "properties": { + "provenance": { "type": ["object", "null"] }, + "source_check": { + "type": "object", + "description": "A live check of the bound exports against their \ + declared sources.", + "properties": { + "outcome": { "type": "string" }, + "checked_at": { "type": "string", "format": "date-time" }, + "data_as_of": { "type": "string", "format": "date-time" }, + "datamk_version": { "type": "string" }, + "exports": { + "type": "object", + "description": "Route key -> what the check measured. \ + Timestamped by `checked_at`.", + "additionalProperties": { "type": "object", + "required": ["check", "grain", "rows", "distinct_grain"], + "properties": { + "check": { "type": "string", "enum": ["grain_unique"] }, + "grain": { "type": "array", + "items": { "type": "string" } }, + "rows": { "type": "integer" }, + "distinct_grain": { "type": "integer" } + }} + } + } + }, + "freshness": { "type": "object" }, + "upstreams": { "type": "array", "items": { "type": "object" } }, + "exports": { + "type": "object", + "description": "Route key -> swap-time probe (rows, coverage, values, \ + example_request).", + "additionalProperties": { "type": "object" } + }, + "docs": { "type": "object", "additionalProperties": { "type": "object", + "properties": { + "sha256": { "type": "string" }, + "bytes": { "type": "integer" } + }}}, + "source_descriptions": { + "type": "object", + "description": "Source name (as declared under `sources:`) -> column \ + -> upstream description. Keyed by source, NOT by \ + export; observed, so it may disagree with the \ + declared schema.", + "additionalProperties": { "type": "object", + "additionalProperties": { "type": "string" } } + } + } + }, + "data": { + "type": "object", + "required": ["served_here", "channels"], + "properties": { + "served_here": { "type": "boolean", + "description": "False when this endpoint serves no rows — see each \ + export's `binding` for where they are." }, + "channels": { "type": "array", "items": { "type": "string" }, + "description": "Operator hints from the profile. Free-form prose; \ + `declared.exports[].binding` is the machine-readable \ + target." } + } + }, + "notes": { "type": "array", "items": { "type": "string" } }, + "included": { + "type": "array", + "items": { "type": "string", "enum": super::INCLUDE_SECTIONS.iter() + .map(|s| json!(s)) + .collect::>() }, + "description": "Section NAMES inlined by this response, never their content. \ + Each name appears as a top-level key of the same name." + }, + "docs": { + "type": "object", + "description": "Present only when `included` contains `docs`. Keyed by \ + `declared.docs[].target`.", + "additionalProperties": { "type": "object", + "required": ["media_type", "content"], + "properties": { + "media_type": { "type": "string" }, + "content": { "type": "string" } + }} + } + } + }) +} + +fn declared_export_schema() -> Value { + json!({ + "type": "object", + "required": ["name", "version", "route", "contract", "grain", "schema"], + "properties": { + "name": { "type": "string" }, + "version": { "type": "string", "description": "Semver. The route keys on MAJOR." }, + "route": { "type": "string", + "description": "The route key (`name@major`) — this export's identity and its \ + docs `target`. It is also the HTTP path only when `query` is \ + non-null; a bound export has no path." }, + "contract": { "type": "string", "enum": ["experimental", "supported"] }, + "description": { "type": "string" }, + "freshness": { "type": "string" }, + "grain": { "type": "array", "items": { "type": "string" } }, + "schema": { "type": "object", "additionalProperties": { "type": "object", + "properties": { + "type": { "type": "string" }, + "unit": { "type": "string" }, + "description": { "type": "string" } + }}}, + "query": { + "type": ["object", "null"], + "description": "The served query grammar. Null iff this export is bound — the \ + machine-checkable signal that `GET /{route}` does not exist." + }, + "binding": { + "type": "object", + "description": "Where the rows are, for a bound export. Present iff `query` \ + is null. Values are verbatim cell.yaml — never \ + profile-resolved, so a templated table ships as written.", + "required": ["source"], + "properties": { + "source": { "type": "string", "description": "The `sources:` key." }, + "object": { "type": "string", + "description": "The declared warehouse object, or file/glob." }, + "connection": { "type": "string", + "description": "Connection alias; what it resolves to is \ + profile, not contract." } + } + } + } + }) +} + fn openapi_path_item() -> Value { json!({ "get": { @@ -145,6 +339,10 @@ fn path_item(export: &Export) -> Value { } json!({ + // The version and contract used to exist only as prose inside + // `summary` — unreadable to the machines this document is for. + "x-datamk-version": export.version, + "x-datamk-contract": export.contract, "get": { "summary": format!("{} v{}", export.name, export.version), "parameters": params, @@ -287,6 +485,73 @@ mod tests { } } + /// The `/context` 200 used to be a bare description string, so nothing in + /// the spec said the document had a `docs` key, a `binding`, or where + /// `include=` content lands. Pinned to `ContextDocument`'s own field list + /// so a new top-level key can't ship undocumented. + #[test] + fn context_schema_names_every_top_level_document_key() { + let doc = generate("orders", None, &[], "digest123"); + let schema = &doc["paths"]["/context"]["get"]["responses"]["200"]["content"] + ["application/json"]["schema"]; + let props = schema["properties"].as_object().unwrap(); + for key in [ + "datamk_context", + "cell", + "status", + "grain_verified", + "declared", + "observed", + "data", + "notes", + "included", + "docs", + ] { + assert!(props.contains_key(key), "undocumented top-level key {key}"); + } + assert_eq!( + schema["properties"]["datamk_context"]["const"], + json!(crate::context::DATAMK_CONTEXT_VERSION) + ); + // The two fields the reported round-trips were lost on. + let export = &props["declared"]["properties"]["exports"]["items"]["properties"]; + assert!(export.get("binding").is_some()); + let docs_entry = &props["declared"]["properties"]["docs"]["items"]["properties"]; + assert!(docs_entry.get("source_path").is_some()); + assert!( + docs_entry.get("path").is_none(), + "the v1 name must not linger in the spec" + ); + // The include= landing rule an agent otherwise learns by trial. + let include_desc = doc["paths"]["/context"]["get"]["parameters"][0]["description"] + .as_str() + .unwrap(); + assert!(include_desc.contains("TOP-LEVEL key"), "{include_desc}"); + } + + /// Version and contract used to exist only as prose inside `summary`. + #[test] + fn data_path_items_carry_version_and_contract_as_extensions() { + let def = CellDef { + cell: "orders".to_string(), + description: None, + docs: None, + sources: IndexMap::new(), + transforms: vec![], + interface: vec![export_with( + "orders_daily", + "2.1.0", + Visibility::Discoverable, + )], + access: Default::default(), + }; + let routes = crate::context::discoverable_routes(&def).unwrap(); + let doc = generate(&def.cell, None, &routes, "digest123"); + let item = &doc["paths"]["/orders_daily@2"]; + assert_eq!(item["x-datamk-version"], "2.1.0"); + assert_eq!(item["x-datamk-contract"], "experimental"); + } + #[test] fn grain_param_without_a_declared_type_makes_no_type_claim() { let mut e = export_with("orders_daily", "2.1.0", Visibility::Discoverable); diff --git a/src/verify.rs b/src/verify.rs index 0c101ed..55a123c 100644 --- a/src/verify.rs +++ b/src/verify.rs @@ -1,6 +1,6 @@ use anyhow::{bail, Context, Result}; use duckdb::Connection; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::Path; use crate::config::{CellDef, MaterializeStrategy, ResolvedTransform}; @@ -319,9 +319,9 @@ pub fn run(file: &Path, profile: &str) -> Result<()> { } else { HashMap::new() }; - check(&cell.conn, &cell.def, &warehouse_columns)?; + let measurements = check(&cell.conn, &cell.def, &warehouse_columns)?; if has_bound_exports { - write_source_check_record(file, &cell.dir, profile) + write_source_check_record(file, &cell.dir, profile, measurements) .context("writing the live-verify source-check record (.cell/source_check.json)")?; // Issue #6/#10: the same live bind pass above already carries // `warehouse_columns` — persisted here under the identical @@ -359,7 +359,12 @@ pub fn run(file: &Path, profile: &str) -> Result<()> { /// `data_as_of` stays `None` in this slice — no connector currently threads /// a cheap, truthful "as of" timestamp out of the bind path; fabricating one /// (or defaulting it to `checked_at`) is exactly what ADR 0012 §2 forbids. -fn write_source_check_record(file: &Path, dir: &Path, profile: &str) -> Result<()> { +fn write_source_check_record( + file: &Path, + dir: &Path, + profile: &str, + exports: BTreeMap, +) -> Result<()> { let path = dir.join(".cell").join("source_check.json"); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent) @@ -373,6 +378,7 @@ fn write_source_check_record(file: &Path, dir: &Path, profile: &str) -> Result<( datamk_version: env!("CARGO_PKG_VERSION").to_string(), cell_yaml_digest, profile: profile.to_string(), + exports, }; std::fs::write(&path, serde_json::to_string_pretty(&record)?) .with_context(|| format!("writing {}", path.display()))?; @@ -405,11 +411,15 @@ fn write_source_check_record(file: &Path, dir: &Path, profile: &str) -> Result<( /// connector with no classification job — not a lesser fallback, there is /// genuinely no other authority to consult there either). Mixed cells hit /// both paths in the same loop. +/// Returns what the grain check measured, per route key — the numbers behind +/// a passing check, for `.cell/source_check.json` and from there +/// `observed.source_check.exports`. A grainless export contributes nothing: +/// no check ran on it. pub fn check( conn: &Connection, def: &CellDef, warehouse_columns: &HashMap, -) -> Result<()> { +) -> Result> { // ADR 0005 §1: `__datamk_` is a reserved, enforced namespace — a table // matching it other than the watermark table itself is refused before // publish. @@ -423,6 +433,7 @@ pub fn check( // mandatory — an experimental export needs nothing. check_supported_have_descriptions(def)?; + let mut measurements = BTreeMap::new(); for export in &def.interface { let source = export .bind @@ -512,11 +523,20 @@ pub fn check( export.grain ); } + measurements.insert( + export.route()?, + crate::manifest::GrainMeasurement { + check: "grain_unique".to_string(), + grain: export.grain.clone(), + rows: total, + distinct_grain: distinct, + }, + ); } tracing::info!(export = %export.name, version = %export.version, "interface ok"); } - Ok(()) + Ok(measurements) } /// ADR 0012 §3 ratchet check 4: an export with `contract: supported` must diff --git a/test/integrations/cli.rs b/test/integrations/cli.rs index a17eee6..e981717 100644 --- a/test/integrations/cli.rs +++ b/test/integrations/cli.rs @@ -830,7 +830,7 @@ fn context_emits_a_draft_document_for_a_local_cell() { let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("context emits valid JSON on stdout"); - assert_eq!(doc["datamk_context"], 1); + assert_eq!(doc["datamk_context"], 2); assert_eq!(doc["cell"], "ctxcell"); assert_eq!(doc["status"], "draft", "pinless => draft, by definition"); assert_eq!(doc["grain_verified"], false);