From 661ff24e443c38300bdddd6d8792967de61917ca Mon Sep 17 00:00:00 2001 From: bioedca <131809205+bioedca@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:23:30 -0500 Subject: [PATCH 1/7] test(docs): pin the validation-page numbers to their source artifacts docs/validation.md is a transcription page: it restates the four $.tolerance bounds, the four $.tolerance_by_method.ebhmm bounds, both pooled_worst blocks, the kinSoftChallenge rate matrices and roughly a dozen module constants from src/. Nothing pinned any of them, so a re-freeze or a constant edit left the published validation page silently wrong while mkdocs build --strict, which is a link-and-nav check rather than a value check, stayed green. That is the failure mode #158 was written to eliminate. tests/test_docs_validation_numbers.py closes it. Each pin locates a region structurally before it reads a number - a table selected by a header cell then a row by its label and a column by its header, or the one non-table paragraph of a section that names an anchor token - and every step asserts a unique match, so a page edit that duplicates or renames a table, row or column fails loudly instead of silently widening what counts. Inside a paragraph a literal belongs to the nearest name before it, which is how the page writes every one of them, so moving a number next to a neighbouring key fails rather than passing because the digits are still present. Artifacts are read with json and module constants with ast, so the guard imports no tether and runs unmarked on the 3-OS matrix. Figures with no committed source - upstream commit dates and change counts, historical Actions runs, and ADR-0022's local-run numbers - are enumerated in EXCLUSIONS with a reason each, because an unexplained omission on a validation page is indistinguishable from an oversight. Closes #214. --- tests/test_docs_validation_numbers.py | 929 ++++++++++++++++++++++++++ 1 file changed, 929 insertions(+) create mode 100644 tests/test_docs_validation_numbers.py diff --git a/tests/test_docs_validation_numbers.py b/tests/test_docs_validation_numbers.py new file mode 100644 index 00000000..202d3c1a --- /dev/null +++ b/tests/test_docs_validation_numbers.py @@ -0,0 +1,929 @@ +# SPDX-FileCopyrightText: 2026 The Tether Authors +# SPDX-License-Identifier: GPL-3.0-or-later +"""Every number ``docs/validation.md`` transcribes still matches its source artifact. + +``docs/validation.md`` is a *transcription* page: it restates the four ``$.tolerance`` +bounds, the four ``$.tolerance_by_method.ebhmm`` bounds, both ``pooled_worst`` blocks, +the kinSoftChallenge rate matrices and roughly a dozen module constants from ``src/``. +Nothing pinned any of them. ``mkdocs build --strict`` is a link-and-nav check, not a +value check, so a freeze that moved a bound — or a constant edited in ``src/`` — left +the published validation page silently wrong while every gate stayed green. That is the +exact failure mode #158 was written to eliminate, so this module closes it (#214). + +**Structural selection, not literal grepping.** A pin never searches the page for its +own number; it *locates a region first* and then asks that region what it states: + +* :class:`Cell` selects the table in a named ``##`` section whose header carries a given + cell, then the row whose first cell names the pin, then the column by its header text. + Every step asserts a unique match, so a page edit that duplicates or renames a table, + row or column fails loudly here instead of silently widening what counts. +* :class:`Prose` selects the one non-table paragraph of a section that names an anchor + token, then attributes each literal in it to the nearest *preceding* name token. So + ``` `viterbi_agreement` 0.9355949176941853 ``` binds that number to that key, and + moving the number next to a different key breaks the pin rather than passing because + the digits are still somewhere on the page. +* :class:`Quote` renders a live artifact value into a fixed phrase and requires the + phrase verbatim in the selected paragraph. It carries the two figures the page prints + as percentages and the two DOIs. + +The artifacts are read with :mod:`json` and the module constants with :mod:`ast`, so the +guard is dependency-free: no ``tether`` import, no SciPy/h5py, and it runs unmarked on +the plain 3-OS ``test`` matrix. + +**What is deliberately not pinned** is enumerated in :data:`EXCLUSIONS` with a reason +for each, because an unexplained omission on a validation page is indistinguishable from +an oversight. In short: figures whose source is upstream git history, a historical +Actions run, or a prose record of a local run that produced no machine-readable artifact. +""" + +from __future__ import annotations + +import ast +import contextlib +import json +import re +from pathlib import Path +from typing import NamedTuple + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +PAGE = ROOT / "docs" / "validation.md" +SRC = ROOT / "src" +PARITY_PATH = ROOT / "schema" / "parity_tolerance.json" +KINSOFT_PATH = ROOT / "schema" / "kinsoft_reference.json" + +PAGE_TEXT = PAGE.read_text(encoding="utf-8") +PARITY = json.loads(PARITY_PATH.read_text(encoding="utf-8")) +KINSOFT = json.loads(KINSOFT_PATH.read_text(encoding="utf-8")) + + +# --- Reading the source artifacts --------------------------------------------- + + +def at(doc: dict, pointer: str) -> object: + """The value of a dotted ``$.a.b.c`` pointer, spelled the way the page spells it.""" + node: object = doc + for key in pointer.removeprefix("$.").split("."): + assert isinstance(node, dict), f"{pointer}: {key!r} is not under a mapping" + assert key in node, f"{pointer}: no key {key!r}" + node = node[key] + return node + + +def module_constant(dotted: str, name: str) -> object: + """The module-level value of ``name`` in ``tether.x.y``, read from disk with ``ast``. + + Parsed rather than imported so the guard has no runtime dependency and executes no + module-import side effects — the same reason ``test_docs_parameter_defaults`` reads + its defaults this way. + """ + path = SRC / Path(*dotted.split(".")).with_suffix(".py") + for stmt in ast.parse(path.read_text(encoding="utf-8"), filename=str(path)).body: + if isinstance(stmt, ast.AnnAssign): + if isinstance(stmt.target, ast.Name) and stmt.target.id == name: + assert stmt.value is not None + return ast.literal_eval(stmt.value) + elif isinstance(stmt, ast.Assign): + for target in stmt.targets: + if isinstance(target, ast.Name) and target.id == name: + return ast.literal_eval(stmt.value) + raise AssertionError(f"{dotted} has no module-level constant {name!r}") + + +# --- Reading the page ---------------------------------------------------------- + +_HEADING_RE = re.compile(r"^## (.+)$") +_SEPARATOR_RE = re.compile(r"^\|[\s:|-]+\|$") +_BACKTICKED_RE = re.compile(r"`([^`]+)`") +_QUOTED_RE = re.compile(r'"([^"]*)"') +_NUMBER_RE = re.compile(r"[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?") +#: A constant as the page spells one. Leading-underscore test helpers (``_EXPECTED_ +#: EVIDENCE``) are deliberately out: the page names them, it never states their value. +#: Paired with the backtick filter in +#: :func:`test_every_constant_the_page_states_a_value_for_is_pinned`, because prose +#: acronyms (``PRD §11.2``, ``PEP 610``, ``RMS ≤ 0.5 px``) are shaped like constants and +#: sit next to a number without being one; the page only ever writes real code in code +#: spans. +_CONSTANT_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") + +#: Name / string / number, tried in that order so ``smd_281mol`` and ``k12`` stay whole +#: names rather than decomposing into a name and a stray number. Slashes and hyphens are +#: part of a name so ``src/tether/fret/gamma.py`` and ``ADR-0043`` do not leak digits. +_TOKEN_RE = re.compile( + r"""(?P[A-Za-z_$][\w.$/-]*) + | (?P"[^"]*") + | (?P[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)""", + re.VERBOSE, +) + + +class Table(NamedTuple): + header: tuple[str, ...] + rows: tuple[tuple[str, ...], ...] + + +def _cells(line: str) -> list[str]: + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def _sections(text: str) -> dict[str, str]: + """The page split on its ``##`` headings. Prose above the first one is dropped.""" + out: dict[str, str] = {} + name: str | None = None + buf: list[str] = [] + for line in text.split("\n"): + heading = _HEADING_RE.match(line) + if heading: + if name is not None: + out[name] = "\n".join(buf) + name, buf = heading.group(1).strip(), [] + else: + buf.append(line) + if name is not None: + out[name] = "\n".join(buf) + return out + + +def _tables(body: str) -> list[Table]: + """Every Markdown table in ``body``, header row first.""" + out: list[Table] = [] + header: list[str] | None = None + rows: list[tuple[str, ...]] = [] + for line in body.split("\n"): + stripped = line.strip() + if stripped.startswith("|"): + if _SEPARATOR_RE.match(stripped): + continue + if header is None: + header = _cells(stripped) + else: + rows.append(tuple(_cells(stripped))) + elif header is not None: + out.append(Table(tuple(header), tuple(rows))) + header, rows = None, [] + if header is not None: + out.append(Table(tuple(header), tuple(rows))) + return out + + +def _prose_paragraphs(body: str) -> list[str]: + """Blank-line-separated blocks that are not tables. + + Tables are excluded because several anchors (``$.tolerance_by_method.ebhmm``) name + both a sentence and a column header; a :class:`Prose` pin must never resolve to the + table a :class:`Cell` pin already owns. + """ + blocks = [b for b in re.split(r"\n\s*\n", body) if b.strip()] + return [ + b + for b in blocks + if not all(line.strip().startswith("|") for line in b.split("\n") if line.strip()) + ] + + +def _owned_literals(paragraph: str) -> dict[str, set[object]]: + """Each name token in ``paragraph`` mapped to the literals that follow it. + + A literal belongs to the nearest name *before* it, which is how this page writes + every one of them — ``` `k32` 0.680 ``` , ``` `measured_utc` `"2026-07-08"` ``` . + Backticks are stripped first so a whole backticked clause (``DEFAULT_SHIP_BAR_PTS = + 10.0``) tokenizes like the prose around it. + """ + owner: str | None = None + out: dict[str, set[object]] = {} + for match in _TOKEN_RE.finditer(paragraph.replace("`", " ")): + kind = match.lastgroup + text = match.group() + if kind == "name": + owner = text + elif owner is not None: + with contextlib.suppress(SyntaxError, ValueError): + out.setdefault(owner, set()).add(ast.literal_eval(text)) + return out + + +def _states(cell: str, value: object) -> bool: + """Does the table cell ``cell`` state ``value``? + + A string is compared against the whole cell and against each backticked or quoted + token in it, because the page writes ``2026-06-26`` bare in a table and + ``` `"2026-06-26"` ``` in prose. A number is compared against every numeric literal + in the cell — which is what lets ``≤ 0.12`` state ``0.12`` — with the parsed type + required to match, so ``0`` never satisfies ``0.0``. + """ + if isinstance(value, str): + candidates = {cell.strip(), cell.strip().strip("`").strip('"')} + candidates |= set(_BACKTICKED_RE.findall(cell)) | set(_QUOTED_RE.findall(cell)) + return value in candidates + for text in _NUMBER_RE.findall(cell): + try: + parsed = ast.literal_eval(text) + except (SyntaxError, ValueError): # pragma: no cover - defensive + continue + if type(parsed) is type(value) and parsed == value: + return True + return False + + +# --- The three pin kinds ------------------------------------------------------- + + +class Cell(NamedTuple): + """One value read out of a named table row and column.""" + + section: str + #: A header cell that identifies the table uniquely inside ``section``. + table: str + #: The row, matched against its first cell's backticked tokens or as a prefix. + row: str + #: The header cell of the column holding the value. + column: str + #: Where the live value comes from, for the failure message. + source: str + value: object + + @property + def label(self) -> str: + return f"{self.row}|{self.column}" + + +class Prose(NamedTuple): + """One value stated next to its own key in a sentence.""" + + section: str + #: A backticked token naming the one non-table paragraph this pin reads. + anchor: str + #: The name token the value must sit behind. + key: str + source: str + value: object + + @property + def label(self) -> str: + return f"{self.anchor}|{self.key}" + + +class Quote(NamedTuple): + """A live value rendered into a fixed phrase that must appear verbatim.""" + + section: str + anchor: str + #: ``str.format`` template; ``{}`` takes the rendered value. + template: str + source: str + value: object + #: Multiplier applied before rendering — the page prints two rates as percentages. + scale: float = 1.0 + + @property + def label(self) -> str: + return f"{self.anchor}|{self.template}" + + def rendered(self) -> str: + if isinstance(self.value, str): + return self.template.format(self.value) + return self.template.format(f"{self.value * self.scale:g}") + + +Pin = Cell | Prose | Quote + + +# --- Locating a pin's region --------------------------------------------------- + + +def _section(text: str, name: str) -> str: + sections = _sections(text) + assert name in sections, f"docs/validation.md has no '## {name}' section" + return sections[name] + + +def _table(text: str, pin: Cell) -> Table: + candidates = [t for t in _tables(_section(text, pin.section)) if pin.table in t.header] + assert len(candidates) == 1, ( + f"'{pin.section}' must hold exactly one table with a {pin.table!r} header cell; " + f"found {len(candidates)}" + ) + return candidates[0] + + +def cell_text(text: str, pin: Cell) -> str: + """The single cell a :class:`Cell` pin addresses.""" + table = _table(text, pin) + assert pin.column in table.header, ( + f"the {pin.table!r} table in '{pin.section}' has no {pin.column!r} column; " + f"its header is {list(table.header)}" + ) + column = table.header.index(pin.column) + matches = [ + row + for row in table.rows + if pin.row in _BACKTICKED_RE.findall(row[0]) or row[0].startswith(pin.row) + ] + assert len(matches) == 1, ( + f"the {pin.table!r} table in '{pin.section}' must hold exactly one row for " + f"{pin.row!r}; found {len(matches)}" + ) + row = matches[0] + assert column < len(row), f"row {pin.row!r} has no cell under {pin.column!r}" + return row[column] + + +def paragraph(text: str, section: str, anchor: str) -> str: + """The one non-table paragraph of ``section`` whose backticked tokens carry ``anchor``.""" + matches = [ + p for p in _prose_paragraphs(_section(text, section)) if anchor in _BACKTICKED_RE.findall(p) + ] + assert len(matches) == 1, ( + f"'{section}' must hold exactly one non-table paragraph naming `{anchor}`; " + f"found {len(matches)} — re-anchor the pin rather than relaxing it" + ) + return matches[0] + + +def satisfied(text: str, pin: Pin) -> bool: + """Does ``text`` still state this pin's live value where the pin says it does?""" + if isinstance(pin, Cell): + return _states(cell_text(text, pin), pin.value) + if isinstance(pin, Prose): + owned = _owned_literals(paragraph(text, pin.section, pin.anchor)).get(pin.key, set()) + return any(type(x) is type(pin.value) and x == pin.value for x in owned) + return pin.rendered() in paragraph(text, pin.section, pin.anchor) + + +# --- The registry -------------------------------------------------------------- +# Every entry reads its value live, so a bound re-frozen in schema/, a rate re-measured +# in schema/, or a constant edited in src/ fails here until the page follows. + +_PARITY_SECTION = "(b) Idealization parity vs tMAVEN" +_KINSOFT_SECTION = "External benchmark" +_BOUNDS = ( + "state_count_min_fraction", + "state_mean_abs_delta_max", + "viterbi_min_agreement", + "relative_elbo_max", +) +_METRICS = ( + "state_count_fraction", + "state_mean_abs_delta", + "viterbi_agreement", + "relative_elbo", +) +#: The kinSoft level-3 rate matrix, in the order the page prints it. +_LEVEL3_RATES = ("k12", "k14", "k21", "k23", "k32", "k41") + + +def _registry() -> list[Pin]: + pins: list[Pin] = [] + + # (a) — the M1 extraction gates, module constants rather than a JSON artifact. + pins += [ + Cell("(a) Extraction vs Deep-LASI", "Role", name, "Value", f"oracle.{name}", value) + for name, value in ( + (n, module_constant("tether.project.oracle", n)) + for n in ("RECALL_THRESHOLD", "MATCH_TOL_PX", "PEARSON_THRESHOLD", "RMS_THRESHOLD_PX") + ) + ] + + # (b) — both frozen tolerance blocks, side by side in one table. + for bound in _BOUNDS: + pins.append( + Cell( + _PARITY_SECTION, + "Bound", + bound, + "`$.tolerance` (default)", + f"$.tolerance.{bound}", + at(PARITY, f"$.tolerance.{bound}"), + ) + ) + pins.append( + Cell( + _PARITY_SECTION, + "Bound", + bound, + "`$.tolerance_by_method.ebhmm`", + f"$.tolerance_by_method.ebhmm.{bound}", + at(PARITY, f"$.tolerance_by_method.ebhmm.{bound}"), + ) + ) + + # (b) — the vbconhmm pooled-worst block, printed at full precision. + pins += [ + Cell( + _PARITY_SECTION, + "Pooled worst case", + metric, + "Pooled worst case", + f"$.pooled_worst.{metric}", + at(PARITY, f"$.pooled_worst.{metric}"), + ) + for metric in _METRICS + ] + + # (b) — the ebFRET pooled-worst block, printed in a sentence instead of a table. + ebhmm = "$.measured_by_method.ebhmm" + pins += [ + Prose( + _PARITY_SECTION, + ebhmm, + metric, + f"{ebhmm}.pooled_worst.{metric}", + at(PARITY, f"{ebhmm}.pooled_worst.{metric}"), + ) + for metric in _METRICS + ] + pins.append( + Prose( + _PARITY_SECTION, + ebhmm, + "measured_utc", + f"{ebhmm}.measured_utc", + at(PARITY, f"{ebhmm}.measured_utc"), + ) + ) + + # (b) — the artifact's own provenance, both as prose and in the build table. + pins += [ + Prose( + _PARITY_SECTION, + "$.tolerance_by_method.ebhmm", + key, + f"$.{key}", + at(PARITY, f"$.{key}"), + ) + for key in ("schema_version", "frozen_at_milestone", "measured_utc") + ] + # The build table sets the two blocks side by side. Note the asymmetry it encodes: + # the M0.5 date is the artifact's *top-level* `measured_utc` (the vbconhmm block has + # none of its own), while the ebFRET one hangs off `measured_by_method`. + m05_column = "`$.method` (M0.5, vbconhmm)" + ebhmm_column = "`$.measured_by_method.ebhmm.method`" + pins += [ + Cell(_PARITY_SECTION, m05_column, row, column, source, value) + for row, column, source, value in ( + ("measured_utc", m05_column, "$.measured_utc", PARITY["measured_utc"]), + ( + "measured_utc", + ebhmm_column, + f"{ebhmm}.measured_utc", + at(PARITY, f"{ebhmm}.measured_utc"), + ), + ( + "sidecar_python_version", + m05_column, + "$.method.sidecar_python_version", + at(PARITY, "$.method.sidecar_python_version"), + ), + ( + "sidecar_python_version", + ebhmm_column, + f"{ebhmm}.method.sidecar_python_version", + at(PARITY, f"{ebhmm}.method.sidecar_python_version"), + ), + ) + ] + + # (d) and (e) and (g) — the remaining module constants the page prints a value for. + pins.append( + Prose( + "(d) Ranker held-out cross-validation", + "DEFAULT_SHIP_BAR_PTS = 10.0", + "DEFAULT_SHIP_BAR_PTS", + "ml.prequential.DEFAULT_SHIP_BAR_PTS", + module_constant("tether.ml.prequential", "DEFAULT_SHIP_BAR_PTS"), + ) + ) + pins += [ + Cell( + "(e) α / γ estimator edge cases", + "Module", + name, + "Value", + f"{module.rsplit('.', 1)[-1]}.{name}", + module_constant(module, name), + ) + for module, name in ( + ("tether.fret.leakage", "LEAKAGE_CEILING"), + ("tether.fret.leakage", "DEFAULT_MIN_WINDOW_FRAMES"), + ("tether.fret.leakage", "DEFAULT_MIN_QUALIFYING_TRACES"), + ("tether.fret.gamma", "GAMMA_CEILING"), + ("tether.fret.gamma", "DEFAULT_GAMMA_HALF_WINDOW"), + ) + ] + pins += [ + Prose( + "(g) Photobleaching detector", + "PB_PRIOR_MU", + name, + f"photobleach.{name}", + module_constant("tether.fret.photobleach", name), + ) + for name in ("PB_PRIOR_A", "PB_PRIOR_B", "PB_PRIOR_BETA", "PB_PRIOR_MU") + ] + + # External benchmark — the kinSoft artifact's own provenance. + pins += [ + Prose(_KINSOFT_SECTION, "schema/kinsoft_reference.json", key, f"$.{key}", KINSOFT[key]) + for key in ("schema_version", "frozen_at_milestone", "measured_utc") + ] + pins += [ + Quote(_KINSOFT_SECTION, "schema/kinsoft_reference.json", "{}", f"$.source.{key}", value) + for key, value in ( + ("doi", at(KINSOFT, "$.source.doi")), + ("data_doi", at(KINSOFT, "$.source.data_doi")), + ("license", at(KINSOFT, "$.source.license")), + ) + ] + + # External benchmark — the level-1 acquisition parameters. + level1 = "$.levels.level1" + pins.append( + Prose( + _KINSOFT_SECTION, + level1, + level1, + f"{level1}.n_traces", + at(KINSOFT, f"{level1}.n_traces"), + ) + ) + pins += [ + Prose(_KINSOFT_SECTION, level1, key, f"{level1}.{key}", at(KINSOFT, f"{level1}.{key}")) + for key in ("dt_s", "sampling_rate_hz", "snr_estimate") + ] + + # External benchmark — the 2-state ground truth / Tether / band matrix. + for rate in ("k12_low_high", "k21_high_low"): + pins.append( + Cell( + _KINSOFT_SECTION, + "Advisory band", + rate, + "Ground truth (s⁻¹)", + f"{level1}.ground_truth.rates_s_inv.{rate}", + at(KINSOFT, f"{level1}.ground_truth.rates_s_inv.{rate}"), + ) + ) + pins.append( + Cell( + _KINSOFT_SECTION, + "Advisory band", + rate, + "Tether (s⁻¹)", + f"{level1}.tether_measured.{rate}", + at(KINSOFT, f"{level1}.tether_measured.{rate}"), + ) + ) + pins.append( + Cell( + _KINSOFT_SECTION, + "Advisory band", + rate, + "Relative deviation", + f"{level1}.tether_measured.{rate.split('_')[0]}_rel_dev", + at(KINSOFT, f"{level1}.tether_measured.{rate.split('_')[0]}_rel_dev"), + ) + ) + pins.append( + Cell( + _KINSOFT_SECTION, + "Advisory band", + rate, + "Advisory band", + "$.band.rate_rel_deviation_max", + at(KINSOFT, "$.band.rate_rel_deviation_max"), + ) + ) + band = at(KINSOFT, "$.band.rate_rel_deviation_max") + spread = f"{level1}.reported_inter_tool_spread" + pins.append( + Prose( + _KINSOFT_SECTION, + "$.band.rate_rel_deviation_max = 0.12", + "$.band.rate_rel_deviation_max", + "$.band.rate_rel_deviation_max", + band, + ) + ) + pins.append( + Quote( + _KINSOFT_SECTION, + "$.band.rate_rel_deviation_max = 0.12", + "({} % average)", + f"{spread}.rate_mean_rel_dev_from_gt", + at(KINSOFT, f"{spread}.rate_mean_rel_dev_from_gt"), + scale=100, + ) + ) + pins.append( + Quote( + _KINSOFT_SECTION, + "$.band.rate_rel_deviation_max = 0.12", + "≥ {} % (1σ)", + f"{spread}.finite_size_uncertainty_1sigma", + at(KINSOFT, f"{spread}.finite_size_uncertainty_1sigma"), + scale=100, + ) + ) + + # External benchmark — the deferred levels and level 3's full ground-truth matrix. + for level in ("level2", "level3"): + pins.append( + Cell( + _KINSOFT_SECTION, + "Why deferred (`$.levels.*.deferred_reason`)", + level, + "States", + f"$.levels.{level}.nstates", + at(KINSOFT, f"$.levels.{level}.nstates"), + ) + ) + pins.append( + Cell( + _KINSOFT_SECTION, + "Why deferred (`$.levels.*.deferred_reason`)", + level, + "Traces", + f"$.levels.{level}.n_traces", + at(KINSOFT, f"$.levels.{level}.n_traces"), + ) + ) + rates3 = "$.levels.level3.ground_truth.rates_s_inv" + pins += [ + Prose(_KINSOFT_SECTION, rates3, rate, f"{rates3}.{rate}", at(KINSOFT, f"{rates3}.{rate}")) + for rate in _LEVEL3_RATES + ] + pins += [ + Quote(_KINSOFT_SECTION, rates3, f"`{name}` {{}} s⁻¹", f"$.levels.level3 {name}", value) + for name, value in (("kbright", 7), ("kdark", 0.007)) + ] + + return pins + + +REGISTRY = _registry() + + +# --- What is deliberately left unpinned ---------------------------------------- + +#: Figures the page prints that no committed artifact holds, so a pin here would assert +#: nothing (or would assert that two transcriptions of the same prose agree). Recorded +#: rather than dropped: on a validation page an unexplained omission and an oversight +#: look identical, and #214 asks for the reason in the test. +EXCLUSIONS: dict[str, str] = { + "tMAVEN upstream commit hashes and their 2025-05-06 / 2025-10-05 dates": ( + "The dates are upstream-repository facts held in no committed artifact. #214 puts " + "the hashes beside them out of scope; the artifact does in fact record both under " + "`$.method.tmaven_commit` and `$.measured_by_method.ebhmm.method.tmaven_commit`, " + "so pinning them is a separate, filed follow-up rather than a silent widening here." + ), + "'13 upstream commits behind' and the 256 / 42 changed-line counts": ( + "Derived by diffing two upstream tMAVEN commits. Nothing in this repository " + "records them, and re-deriving them in a test would need network access." + ), + "the weekly large-fixtures run of 2026-07-20 and its '7 passed, 4 skipped' summary": ( + "A historical GitHub Actions run. It is evidence that the gated assertion skipped " + "on that date, not a value any artifact carries." + ), + "sidecar-measure dispatch run 28963324581": ( + "A historical GitHub Actions run id. It appears in the artifact only inside the " + "free-text `build_provenance` narrative, not as a machine-readable field." + ), + "the ADR-0022 extraction numbers (0.928, 0.984, 0.982, 0.854, 0.34, ~0.66, ~0.9 GB)": ( + "Measured on a local run on the maintainer's workstation and recorded in " + "docs/adr/0022 as prose. There is no machine-readable artifact, so a pin would " + "only assert that two transcriptions of the same paragraph agree." + ), + "the paper's '14 published analyses', '17 %' and '9-14 %' figures": ( + "Facts of Gotz et al. 2022. kinsoft_reference.json records them only inside the " + "free-text `spread_source` and `deferred_reason` strings, which the page " + "paraphrases rather than quotes, so there is no literal to pin against." + ), +} + + +# --- The guard ------------------------------------------------------------------ + + +def test_page_exists_and_still_has_its_sections() -> None: + assert PAGE.is_file(), f"{PAGE} is missing" + sections = _sections(PAGE_TEXT) + for name in ("(a) Extraction vs Deep-LASI", _PARITY_SECTION, _KINSOFT_SECTION): + assert name in sections, f"docs/validation.md lost its '{name}' section" + + +@pytest.mark.parametrize("pin", REGISTRY, ids=[p.source + "@" + p.label for p in REGISTRY]) +def test_documented_value_matches_its_source_artifact(pin: Pin) -> None: + assert satisfied(PAGE_TEXT, pin), ( + f"docs/validation.md no longer states {pin.value!r} for {pin.source} " + f"at {pin.label!r} in '{pin.section}'. The artifact is the authority: update the " + f"page in the same pull request that moved the value." + ) + + +def test_every_frozen_bound_in_both_blocks_is_pinned() -> None: + """A bound added to either tolerance block may not go undocumented.""" + for pointer in ("$.tolerance", "$.tolerance_by_method.ebhmm"): + block = at(PARITY, pointer) + assert isinstance(block, dict) + pinned = {p.source for p in REGISTRY} + missing = sorted(k for k in block if f"{pointer}.{k}" not in pinned) + assert not missing, f"{pointer} carries unpinned bounds {missing}" + + +def test_every_pooled_worst_metric_in_both_blocks_is_pinned() -> None: + pinned = {p.source for p in REGISTRY} + for pointer in ("$.pooled_worst", "$.measured_by_method.ebhmm.pooled_worst"): + block = at(PARITY, pointer) + assert isinstance(block, dict) + missing = sorted(k for k in block if f"{pointer}.{k}" not in pinned) + assert not missing, f"{pointer} carries unpinned metrics {missing}" + + +def test_every_kinsoft_ground_truth_rate_is_pinned() -> None: + pinned = {p.source for p in REGISTRY} + for pointer in ( + "$.levels.level1.ground_truth.rates_s_inv", + "$.levels.level3.ground_truth.rates_s_inv", + ): + block = at(KINSOFT, pointer) + assert isinstance(block, dict) + missing = sorted(k for k in block if f"{pointer}.{k}" not in pinned) + assert not missing, f"{pointer} carries unpinned rates {missing}" + + +def test_every_constant_the_page_states_a_value_for_is_pinned() -> None: + """The registry may not lag the page. + + Any ``SCREAMING_CASE`` constant the page writes *in a code span* and prints a number + beside has to have an entry, so a new frozen constant added to the page cannot drift + unguarded. Table rows are joined before the scan because a constant sits in the + ``Constant`` cell and its value in the next one. + + The code-span requirement is what separates a constant from an acronym: ``PRD §11.2``, + ``PEP 610`` and ``RMS ≤ 0.5 px`` all read as a capitalised name followed by a number, + and none of them is a symbol anything could be pinned to. + """ + pinned = {p.key for p in REGISTRY if isinstance(p, Prose)} + pinned |= {p.row for p in REGISTRY if isinstance(p, Cell)} + # The leading token of each code span, so `DEFAULT_SHIP_BAR_PTS = 10.0` counts. + code_spans = {span.split()[0] for span in _BACKTICKED_RE.findall(PAGE_TEXT) if span.split()} + regions: list[str] = [] + for body in _sections(PAGE_TEXT).values(): + regions += _prose_paragraphs(body) + regions += [" ".join(row) for table in _tables(body) for row in table.rows] + documented = { + name + for region in regions + for name, values in _owned_literals(region).items() + if values and _CONSTANT_RE.match(name) and name in code_spans + } + missing = sorted(documented - pinned) + assert not missing, ( + "docs/validation.md prints a value for these constants but the registry does not " + f"pin them to the code: {missing}" + ) + + +def test_the_page_quotes_the_artifacts_verbatim() -> None: + """Two claims on the page are explicitly *quotations*, so they must still be exact.""" + assert "never gates main" in at(KINSOFT, "$.description"), ( + "docs/validation.md says `$.description` states 'never gates main' verbatim" + ) + assert at(PARITY, "$.coverage.measured_methods") == ["vbconhmm (vb Consensus HMM)"], ( + "docs/validation.md quotes `$.coverage.measured_methods` as exactly " + '["vbconhmm (vb Consensus HMM)"]' + ) + + +def test_provisional_still_equals_the_frozen_tolerance() -> None: + """The page's central mitigating claim about the M0.5 build gap. + + ``docs/validation.md`` argues the stale-build gap is tolerable because the freeze + *confirmed* the provisional PRD §11.2 defaults rather than widening them — "`$.tolerance` + holds exactly `$.provisional`". If that stops being true the argument is void, so it + is asserted here rather than left as prose. + """ + assert at(PARITY, "$.tolerance") == at(PARITY, "$.provisional") + + +def test_every_exclusion_records_why() -> None: + """#214's third acceptance criterion: no silent omissions.""" + assert EXCLUSIONS, "the exclusion list may not be emptied without saying so" + for figure, reason in EXCLUSIONS.items(): + assert reason.strip(), f"exclusion {figure!r} carries no reason" + + +# --- The guard would notice ---------------------------------------------------- +# Everything above asserts what the page says today. These assert that a drifted value +# would actually FAIL, which is the whole of #214 -- a pin that cannot fail is prose. + + +def _mutate(section: str, old: str, new: str) -> str: + """Replace ``old`` with ``new`` inside one ``##`` section of the page.""" + heading = f"\n## {section}\n" + start = PAGE_TEXT.index(heading) + len(heading) + end = PAGE_TEXT.find("\n## ", start) + end = len(PAGE_TEXT) if end == -1 else end + body = PAGE_TEXT[start:end] + assert body.count(old) == 1, f"{old!r} is not unique inside '{section}'" + return PAGE_TEXT[:start] + body.replace(old, new) + PAGE_TEXT[end:] + + +@pytest.mark.parametrize( + ("section", "old", "new", "source"), + [ + # A tightened ebFRET state-count floor: one digit, in one table cell. + ( + _PARITY_SECTION, + "| `state_count_min_fraction` | floor | 0.9 | 0.5249 |", + "| `state_count_min_fraction` | floor | 0.9 | 0.5250 |", + "$.tolerance_by_method.ebhmm.state_count_min_fraction", + ), + # A pooled-worst value rounded rather than transcribed. + ( + _PARITY_SECTION, + "8.34719832143449e-09", + "8.34719832143e-09", + "$.pooled_worst.state_mean_abs_delta", + ), + # An ebFRET number truncated rather than transcribed. The mutation has to change + # the *double*, not just the digits: a last-place edit inside one ULP parses back + # to the same float, and the same float is the same number. + ( + _PARITY_SECTION, + "`viterbi_agreement` 0.9355949176941853", + "`viterbi_agreement` 0.93559492", + "$.measured_by_method.ebhmm.pooled_worst.viterbi_agreement", + ), + # A module constant the page prints but src/ no longer holds. + ( + "(e) α / γ estimator edge cases", + "| `GAMMA_CEILING` | 5.0 |", + "| `GAMMA_CEILING` | 5.5 |", + "gamma.GAMMA_CEILING", + ), + # A kinSoft ground-truth rate. + ( + _KINSOFT_SECTION, + "| `k21_high_low` | 0.22 |", + "| `k21_high_low` | 0.23 |", + "$.levels.level1.ground_truth.rates_s_inv.k21_high_low", + ), + # One entry of the deferred level-3 matrix. + ( + _KINSOFT_SECTION, + "k32 0.680", + "k32 0.690", + "$.levels.level3.ground_truth.rates_s_inv.k32", + ), + ], +) +def test_a_deliberate_one_value_mismatch_fails( + section: str, old: str, new: str, source: str +) -> None: + """Change exactly one number and the pin that owns it must reject the page.""" + mutated = _mutate(section, old, new) + pins = [p for p in REGISTRY if p.source == source] + assert pins, f"no registry entry sources {source}" + for pin in pins: + assert satisfied(PAGE_TEXT, pin), f"{source} does not hold on the unmutated page" + assert not satisfied(mutated, pin), ( + f"the guard accepted a page that states the wrong value for {source} — " + "the pin is decorative" + ) + + +def test_a_number_moved_to_a_neighbouring_key_fails() -> None: + """Presence is not the test; ownership is. + + Swapping two ebFRET pooled-worst values leaves both literals on the page and both + distinct, so a check that merely looked for the digits would stay green while the + page told a reader ebFRET's Viterbi floor was its state-count worst case. + """ + swapped = _mutate( + _PARITY_SECTION, + "`state_count_fraction` 0.6832740213523132", + "`state_count_fraction` 0.9355949176941853", + ) + swapped_text = swapped.replace( + "`viterbi_agreement` 0.9355949176941853", "`viterbi_agreement` 0.6832740213523132" + ) + for metric in ("state_count_fraction", "viterbi_agreement"): + pin = next( + p for p in REGISTRY if p.source == f"$.measured_by_method.ebhmm.pooled_worst.{metric}" + ) + assert satisfied(PAGE_TEXT, pin) + assert not satisfied(swapped_text, pin), ( + f"{metric} was accepted next to another metric's value" + ) + + +def test_an_ambiguous_anchor_fails_loudly_rather_than_picking_one() -> None: + """Region selection asserts uniqueness, so a page edit cannot silently widen it.""" + duplicated = PAGE_TEXT.replace( + "**Frozen constants.** The ±2-frame tolerance", + "`PB_PRIOR_MU` is also mentioned here.\n\n**Frozen constants.** The ±2-frame tolerance", + ) + with pytest.raises(AssertionError, match="exactly one non-table paragraph"): + paragraph(duplicated, "(g) Photobleaching detector", "PB_PRIOR_MU") From e4b8a541c7ea05c4659652da79bb57dc50d56478 Mon Sep 17 00:00:00 2001 From: bioedca <131809205+bioedca@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:39:53 -0500 Subject: [PATCH 2/7] test(docs): read the blinking rates from the artifact, not from the test The kbright / kdark pins restated 7 and 0.007 as literals in the registry, so they asserted the test module's own copy of two numbers rather than the committed artifact's. A pin that reads nothing is prose, which is the defect this whole module exists to remove. Both are the one pair kinsoft_reference.json keeps inside free text - "blinking kbright=7 s^-1, kdark=0.007 s^-1" in the level-3 ground-truth note - rather than as fields of rates_s_inv, so _blinking_rate reads them with a regex bounded to the key name and asserts rather than guesses when the note stops recording them. Types are preserved: int 7, float 0.007. Adds the matching one-value mutation case, so the new pins are shown to fail rather than only to pass. --- tests/test_docs_validation_numbers.py | 32 +++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test_docs_validation_numbers.py b/tests/test_docs_validation_numbers.py index 202d3c1a..a49125f9 100644 --- a/tests/test_docs_validation_numbers.py +++ b/tests/test_docs_validation_numbers.py @@ -372,6 +372,24 @@ def satisfied(text: str, pin: Pin) -> bool: #: The kinSoft level-3 rate matrix, in the order the page prints it. _LEVEL3_RATES = ("k12", "k14", "k21", "k23", "k32", "k41") +#: ``kbright=7 s^-1`` inside the level-3 ``note``, keyed on the name. +_BLINKING_RE = r"\b{name}\s*=\s*([0-9.]+)" + + +def _blinking_rate(name: str) -> object: + """``kbright`` / ``kdark`` as the level-3 ``note`` records them. + + These two are the one pair the artifact keeps inside free text — *"blinking kbright=7 + s^-1, kdark=0.007 s^-1"* — rather than as fields of ``rates_s_inv``, so they are read + with a regex bounded to the key name instead of a pointer. Restating the literal here + would make the pin assert the test's own copy of the number, and a pin that reads + nothing is prose. + """ + note = str(at(KINSOFT, "$.levels.level3.ground_truth.note")) + match = re.search(_BLINKING_RE.format(name=name), note) + assert match, f"the level-3 note no longer records `{name}=`; re-read it before pinning" + return ast.literal_eval(match.group(1)) + def _registry() -> list[Pin]: pins: list[Pin] = [] @@ -653,9 +671,12 @@ def _registry() -> list[Pin]: Prose(_KINSOFT_SECTION, rates3, rate, f"{rates3}.{rate}", at(KINSOFT, f"{rates3}.{rate}")) for rate in _LEVEL3_RATES ] + note = "$.levels.level3.ground_truth.note" pins += [ - Quote(_KINSOFT_SECTION, rates3, f"`{name}` {{}} s⁻¹", f"$.levels.level3 {name}", value) - for name, value in (("kbright", 7), ("kdark", 0.007)) + Quote( + _KINSOFT_SECTION, rates3, f"`{name}` {{}} s⁻¹", f"{note} {name}", _blinking_rate(name) + ) + for name in ("kbright", "kdark") ] return pins @@ -877,6 +898,13 @@ def _mutate(section: str, old: str, new: str) -> str: "k32 0.690", "$.levels.level3.ground_truth.rates_s_inv.k32", ), + # A blinking rate, which is the one pair read out of the artifact's free text. + ( + _KINSOFT_SECTION, + "`kbright` 7 s⁻¹", + "`kbright` 8 s⁻¹", + "$.levels.level3.ground_truth.note kbright", + ), ], ) def test_a_deliberate_one_value_mismatch_fails( From baa0379ffabdeb8e09fc51845f535d382070d49e Mon Sep 17 00:00:00 2001 From: bioedca <131809205+bioedca@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:52:50 -0500 Subject: [PATCH 3/7] test(docs): close four false-pass holes the Codex review found All four let a real docs/source mismatch through, and each falsified something this module claims about itself. Pin the measurement counts. $.method.n_runs_per_fixture and both n_comparisons are machine-readable, but were neither pinned nor listed in EXCLUSIONS, so a re-measure with a different run count left "20 self-reseeded fits per fixture; 39 recorded comparisons" and "19 cross-seed comparisons" green. The 39 is now summed from spread_by_fixture rather than restated. Reaching that paragraph needed a second anchor form - it carries no pointer, only the bold lead-in **Measured result.** - and the fallback is consulted only when the backticked form matches nothing, so no existing anchor changes meaning. Compare quoted claims with the page. test_the_page_quotes_the_artifacts_verbatim read only the artifacts, never PAGE_TEXT, so rewording "never gates main" to its opposite kept it green. A quotation has two sides; both are now checked, and the separate "records only" claim is asserted separately from the string itself. Bind each DOI to its label. The citation pins rendered a bare {}, so the paper doi and the data doi could swap places and both still be somewhere in the paragraph. Each template now carries the words identifying its field, and Quote compares whitespace-normalised text so the page's hard wrap between "Paper doi" and the DOI cannot break the binding. Reject duplicate sections. _sections assigned into a dict, so a stale duplicate of a validation section inserted ABOVE the canonical one was read by nothing and published anyway. A repeated ## heading is now an error, and the heading regex no longer parses a future ### sub-heading as a section named "# ...". Each fix carries the negative test that would have caught it. --- tests/test_docs_validation_numbers.py | 203 ++++++++++++++++++++++---- 1 file changed, 178 insertions(+), 25 deletions(-) diff --git a/tests/test_docs_validation_numbers.py b/tests/test_docs_validation_numbers.py index a49125f9..9261b86a 100644 --- a/tests/test_docs_validation_numbers.py +++ b/tests/test_docs_validation_numbers.py @@ -22,9 +22,16 @@ ``` `viterbi_agreement` 0.9355949176941853 ``` binds that number to that key, and moving the number next to a different key breaks the pin rather than passing because the digits are still somewhere on the page. -* :class:`Quote` renders a live artifact value into a fixed phrase and requires the - phrase verbatim in the selected paragraph. It carries the two figures the page prints - as percentages and the two DOIs. +* :class:`Quote` renders a live artifact value into a fixed phrase and requires the phrase + verbatim in the selected paragraph, whitespace-normalised so a re-wrap cannot break it. + The phrase carries the words that say *which* field the value is, so swapping the paper + DOI with the data DOI fails even though both remain present. It also pins the + measurement counts, the licence, and the two figures the page prints as percentages. + +Section, table, row, column and paragraph selection each assert a **unique** match, and a +repeated ``##`` heading is an error rather than a last-one-wins overwrite — a stale +duplicate section inserted above the canonical one would otherwise be read by nothing here +and published anyway. The artifacts are read with :mod:`json` and the module constants with :mod:`ast`, so the guard is dependency-free: no ``tether`` import, no SciPy/h5py, and it runs unmarked on @@ -93,7 +100,9 @@ def module_constant(dotted: str, name: str) -> object: # --- Reading the page ---------------------------------------------------------- -_HEADING_RE = re.compile(r"^## (.+)$") +#: A level-2 heading only. The negative lookahead keeps a future ``### Sub-heading`` from +#: parsing as a section literally named ``# Sub-heading``. +_HEADING_RE = re.compile(r"^## (?!#)(.+)$") _SEPARATOR_RE = re.compile(r"^\|[\s:|-]+\|$") _BACKTICKED_RE = re.compile(r"`([^`]+)`") _QUOTED_RE = re.compile(r'"([^"]*)"') @@ -128,20 +137,35 @@ def _cells(line: str) -> list[str]: def _sections(text: str) -> dict[str, str]: - """The page split on its ``##`` headings. Prose above the first one is dropped.""" + """The page split on its ``##`` headings. Prose above the first one is dropped. + + A repeated heading is an error, not a last-one-wins overwrite. A stale duplicate of + a validation section inserted *before* the canonical one would otherwise be invisible + to every pin here — they would all read the later copy — while the contradictory + earlier copy stayed published and the suite stayed green. + """ out: dict[str, str] = {} name: str | None = None buf: list[str] = [] + + def close() -> None: + assert name is not None + assert name not in out, ( + f"docs/validation.md has two '## {name}' sections. Every pin would read only " + "the later one while the earlier stayed published; de-duplicate the page." + ) + out[name] = "\n".join(buf) + for line in text.split("\n"): heading = _HEADING_RE.match(line) if heading: if name is not None: - out[name] = "\n".join(buf) + close() name, buf = heading.group(1).strip(), [] else: buf.append(line) if name is not None: - out[name] = "\n".join(buf) + close() return out @@ -330,17 +354,35 @@ def cell_text(text: str, pin: Cell) -> str: def paragraph(text: str, section: str, anchor: str) -> str: - """The one non-table paragraph of ``section`` whose backticked tokens carry ``anchor``.""" - matches = [ - p for p in _prose_paragraphs(_section(text, section)) if anchor in _BACKTICKED_RE.findall(p) - ] + """The one non-table paragraph of ``section`` selected by ``anchor``. + + ``anchor`` is normally a backticked token, matched by equality so that + ``$.levels.level1`` never resolves through ``$.levels.level1.ground_truth``. A few + paragraphs the page never labels with a pointer — ``**Measured result.**`` — are + reachable by their bold lead-in instead, and that fallback is only consulted when the + token form matched nothing, so no existing anchor changes meaning. + """ + paragraphs = _prose_paragraphs(_section(text, section)) + matches = [p for p in paragraphs if anchor in _BACKTICKED_RE.findall(p)] + if not matches: + matches = [p for p in paragraphs if p.lstrip().startswith(anchor)] assert len(matches) == 1, ( - f"'{section}' must hold exactly one non-table paragraph naming `{anchor}`; " + f"'{section}' must hold exactly one non-table paragraph selected by {anchor!r}; " f"found {len(matches)} — re-anchor the pin rather than relaxing it" ) return matches[0] +def _flat(text: str) -> str: + """One line, single-spaced — so a pinned phrase survives Markdown re-wrapping. + + The page hard-wraps at 90-odd columns, so ``Paper doi`` and the DOI that belongs to it + sit on different lines. Without this a label-bound phrase would be unmatchable purely + because of where the paragraph happened to break. + """ + return " ".join(text.split()) + + def satisfied(text: str, pin: Pin) -> bool: """Does ``text`` still state this pin's live value where the pin says it does?""" if isinstance(pin, Cell): @@ -348,7 +390,7 @@ def satisfied(text: str, pin: Pin) -> bool: if isinstance(pin, Prose): owned = _owned_literals(paragraph(text, pin.section, pin.anchor)).get(pin.key, set()) return any(type(x) is type(pin.value) and x == pin.value for x in owned) - return pin.rendered() in paragraph(text, pin.section, pin.anchor) + return _flat(pin.rendered()) in _flat(paragraph(text, pin.section, pin.anchor)) # --- The registry -------------------------------------------------------------- @@ -461,6 +503,35 @@ def _registry() -> list[Pin]: ) ) + # (b) — how much measurement stands behind those blocks. These are machine-readable + # fields, so leaving them neither pinned nor excluded would let a re-measure with a + # different run count keep the page's "20 fits / 39 comparisons" sentence green. + vbconhmm_spread = at(PARITY, "$.spread_by_fixture") + assert isinstance(vbconhmm_spread, dict) + pins += [ + Quote( + _PARITY_SECTION, + "**Measured result.**", + "{} self-reseeded fits per fixture", + "$.method.n_runs_per_fixture", + at(PARITY, "$.method.n_runs_per_fixture"), + ), + Quote( + _PARITY_SECTION, + "**Measured result.**", + "{} recorded comparisons pooled", + "sum of $.spread_by_fixture.*.n_comparisons", + sum(fixture["n_comparisons"] for fixture in vbconhmm_spread.values()), + ), + Quote( + _PARITY_SECTION, + ebhmm, + "{} cross-seed comparisons", + f"{ebhmm}.spread_by_fixture.smd_281mol.n_comparisons", + at(PARITY, f"{ebhmm}.spread_by_fixture.smd_281mol.n_comparisons"), + ), + ] + # (b) — the artifact's own provenance, both as prose and in the build table. pins += [ Prose( @@ -545,12 +616,15 @@ def _registry() -> list[Pin]: Prose(_KINSOFT_SECTION, "schema/kinsoft_reference.json", key, f"$.{key}", KINSOFT[key]) for key in ("schema_version", "frozen_at_milestone", "measured_utc") ] + # Each citation carries the words that identify *which* field it is. A bare `{}` would + # let the paper doi and the data doi swap places on the page and still pass, because + # both values would still be somewhere in the same paragraph. pins += [ - Quote(_KINSOFT_SECTION, "schema/kinsoft_reference.json", "{}", f"$.source.{key}", value) - for key, value in ( - ("doi", at(KINSOFT, "$.source.doi")), - ("data_doi", at(KINSOFT, "$.source.data_doi")), - ("license", at(KINSOFT, "$.source.license")), + Quote(_KINSOFT_SECTION, "schema/kinsoft_reference.json", template, f"$.source.{key}", value) + for key, template, value in ( + ("doi", "Paper doi `{}`", at(KINSOFT, "$.source.doi")), + ("data_doi", "data doi `{}`", at(KINSOFT, "$.source.data_doi")), + ("license", "({})", at(KINSOFT, "$.source.license")), ) ] @@ -806,17 +880,55 @@ def test_every_constant_the_page_states_a_value_for_is_pinned() -> None: ) -def test_the_page_quotes_the_artifacts_verbatim() -> None: - """Two claims on the page are explicitly *quotations*, so they must still be exact.""" - assert "never gates main" in at(KINSOFT, "$.description"), ( - "docs/validation.md says `$.description` states 'never gates main' verbatim" +#: Claims the page presents as *quotations* of an artifact, as +#: ``(fragment, artifact, pointer, section, anchor)``. Each is checked in **both** +#: directions. A one-sided check — the fragment is still in the artifact — passes a page +#: that stopped quoting it, or quotes its opposite; the page saying `$.description` +#: "never gates main" is only true while *both* halves hold. +_VERBATIM_QUOTES = ( + ("never gates main", KINSOFT, "$.description", _KINSOFT_SECTION, "$.description"), + ( + "vbconhmm (vb Consensus HMM)", + PARITY, + "$.coverage.measured_methods", + _PARITY_SECTION, + "$.coverage.measured_methods", + ), +) + + +@pytest.mark.parametrize( + ("fragment", "artifact", "pointer", "section", "anchor"), + _VERBATIM_QUOTES, + ids=[pointer for _, _, pointer, _, _ in _VERBATIM_QUOTES], +) +def test_the_page_quotes_the_artifact_verbatim( + fragment: str, artifact: dict, pointer: str, section: str, anchor: str +) -> None: + """A quotation has two sides, and the page owns one of them.""" + held = at(artifact, pointer) + source = held if isinstance(held, str) else "\n".join(str(item) for item in held) # type: ignore[union-attr] + assert fragment in source, ( + f"docs/validation.md presents {fragment!r} as a verbatim quote of {pointer}, " + "but the artifact no longer says it" ) - assert at(PARITY, "$.coverage.measured_methods") == ["vbconhmm (vb Consensus HMM)"], ( - "docs/validation.md quotes `$.coverage.measured_methods` as exactly " - '["vbconhmm (vb Consensus HMM)"]' + assert fragment in _flat(paragraph(PAGE_TEXT, section, anchor)), ( + f"{pointer} still says {fragment!r} but docs/validation.md no longer quotes it " + "where it claims to" ) +def test_only_one_method_was_measured_as_the_page_says() -> None: + """The page says `$.coverage.measured_methods` records *only* that one method. + + The verbatim check above proves the string is present on both sides; "only" is the + separate claim that nothing joined it, which a second measured method would falsify + while every quotation still matched. + """ + sole = next(q[0] for q in _VERBATIM_QUOTES if q[2] == "$.coverage.measured_methods") + assert at(PARITY, "$.coverage.measured_methods") == [sole] + + def test_provisional_still_equals_the_frozen_tolerance() -> None: """The page's central mitigating claim about the M0.5 build gap. @@ -947,6 +1059,47 @@ def test_a_number_moved_to_a_neighbouring_key_fails() -> None: ) +def test_swapping_the_two_dois_fails() -> None: + """A citation value must carry the words saying *which* field it is. + + Both DOIs stay present and distinct through the swap, so a pin that only asked + whether each value appears somewhere in the paragraph would call the page correct + while it attributed the Zenodo data deposit to the paper. + """ + paper = at(KINSOFT, "$.source.doi") + data = at(KINSOFT, "$.source.data_doi") + swapped = PAGE_TEXT.replace(f"`{paper}`", "@@PAPER@@").replace(f"`{data}`", f"`{paper}`") + swapped = swapped.replace("@@PAPER@@", f"`{data}`") + assert str(paper) in swapped and str(data) in swapped # nothing was lost, only moved + for pin in (p for p in REGISTRY if p.source in ("$.source.doi", "$.source.data_doi")): + assert satisfied(PAGE_TEXT, pin) + assert not satisfied(swapped, pin), f"{pin.source} was accepted under the wrong label" + + +def test_a_stale_duplicate_section_fails_rather_than_being_shadowed() -> None: + """A second copy of a section must not hide behind the canonical one. + + Dictionary assignment used to be last-one-wins, so a stale duplicate inserted *above* + the real section was read by nothing and published anyway. + """ + heading = f"## {_KINSOFT_SECTION}" + duplicated = PAGE_TEXT.replace( + heading, + f"{heading}\n\nA stale copy with `$.band.rate_rel_deviation_max = 0.99`.\n\n{heading}", + 1, + ) + with pytest.raises(AssertionError, match="two '## External benchmark' sections"): + _sections(duplicated) + + +def test_a_dropped_quotation_fails_even_though_the_artifact_still_says_it() -> None: + """The page owns one side of every quotation it presents as verbatim.""" + fragment, artifact, pointer, section, anchor = _VERBATIM_QUOTES[0] + assert fragment in str(at(artifact, pointer)) # the artifact side is untouched + reworded = PAGE_TEXT.replace(f'it "{fragment}"', "it says otherwise") + assert fragment not in _flat(paragraph(reworded, section, anchor)) + + def test_an_ambiguous_anchor_fails_loudly_rather_than_picking_one() -> None: """Region selection asserts uniqueness, so a page edit cannot silently widen it.""" duplicated = PAGE_TEXT.replace( From 3f2da1b6dd392ff6c02f04f998b50f1961c55818 Mon Sep 17 00:00:00 2001 From: bioedca <131809205+bioedca@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:08:48 -0500 Subject: [PATCH 4/7] test(docs): require every restatement to agree, and pin the zero clause Two more false passes from the second Codex read, both of them the same shape as what the module already claims to prevent. A Prose pin accepted the key if ANY literal attributed to it matched, so a paragraph stating one constant twice kept a stale contradictory restatement published as long as one occurrence stayed right - and DEFAULT_SHIP_BAR_PTS is stated twice. Every comparable literal must now agree. Comparable is scoped by KIND rather than exact type: any two numbers are candidates for the same claim, so 11 contradicts 10.0 instead of being waved through, while the nearest-name rule is still allowed to sweep up a trailing count of another kind - "measured_utc "2026-07-08", 19 cross-seed comparisons" attributes both, and only the date is a restatement of measured_utc. Equality itself stays type-exact. The level-3 ground truth lists six non-zero transitions; "all other off-diagonal rates zero" is what makes it a 4x4 matrix. Pinning the six and not the clause let the page say the unlisted transitions were 0.01 with every guard green. Both sides are now checked - the artifact writes "are 0" and the page writes "zero", so the clause is matched by shape and the captured word is then required to MEAN zero - and the enumerated rate set is required to equal the artifact's, without which "all other" no longer covers what it claims to. Each carries the negative test that would have caught it. --- tests/test_docs_validation_numbers.py | 84 ++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/tests/test_docs_validation_numbers.py b/tests/test_docs_validation_numbers.py index 9261b86a..3fbd9377 100644 --- a/tests/test_docs_validation_numbers.py +++ b/tests/test_docs_validation_numbers.py @@ -21,7 +21,9 @@ token, then attributes each literal in it to the nearest *preceding* name token. So ``` `viterbi_agreement` 0.9355949176941853 ``` binds that number to that key, and moving the number next to a different key breaks the pin rather than passing because - the digits are still somewhere on the page. + the digits are still somewhere on the page. *Every* literal attributed to a key has to + agree, not just one, so a key stated twice in one paragraph cannot keep a stale + contradictory restatement alive behind a correct one. * :class:`Quote` renders a live artifact value into a fixed phrase and requires the phrase verbatim in the selected paragraph, whitespace-normalised so a re-wrap cannot break it. The phrase carries the words that say *which* field the value is, so swapping the paper @@ -383,13 +385,37 @@ def _flat(text: str) -> str: return " ".join(text.split()) +def _same_kind(literal: object, value: object) -> bool: + """Could ``literal`` be a *restatement* of ``value``, right or wrong? + + Any two numbers are candidates for the same claim even when their Python types + differ, so ``11`` counts as a contradiction of ``10.0`` rather than being waved + through as a different kind of thing. Equality itself stays type-exact. + """ + numeric = (int, float) + if isinstance(literal, bool) or isinstance(value, bool): + return type(literal) is type(value) + if isinstance(literal, numeric) and isinstance(value, numeric): + return True + return type(literal) is type(value) + + def satisfied(text: str, pin: Pin) -> bool: """Does ``text`` still state this pin's live value where the pin says it does?""" if isinstance(pin, Cell): return _states(cell_text(text, pin), pin.value) if isinstance(pin, Prose): owned = _owned_literals(paragraph(text, pin.section, pin.anchor)).get(pin.key, set()) - return any(type(x) is type(pin.value) and x == pin.value for x in owned) + # EVERY comparable literal must agree, not merely one of them. A key stated twice + # in a paragraph -- `DEFAULT_SHIP_BAR_PTS` is -- would otherwise keep a stale + # contradictory restatement published as long as one occurrence stayed right. + # Scoped by kind because the nearest-name rule legitimately sweeps up a trailing + # count of another kind: "`measured_utc` `"2026-07-08"`, 19 cross-seed comparisons" + # attributes both to `measured_utc`, and only the date is a restatement of it. + comparable = [x for x in owned if _same_kind(x, pin.value)] + return bool(comparable) and all( + type(x) is type(pin.value) and x == pin.value for x in comparable + ) return _flat(pin.rendered()) in _flat(paragraph(text, pin.section, pin.anchor)) @@ -413,6 +439,14 @@ def satisfied(text: str, pin: Pin) -> bool: ) #: The kinSoft level-3 rate matrix, in the order the page prints it. _LEVEL3_RATES = ("k12", "k14", "k21", "k23", "k32", "k41") +_LEVEL3_RATES_POINTER = "$.levels.level3.ground_truth.rates_s_inv" +_LEVEL3_NOTE_POINTER = "$.levels.level3.ground_truth.note" + +#: The clause that turns six numbers into a 4x4 matrix, on either side. The artifact +#: writes "are 0" and the page writes "zero", so it is matched by shape and the captured +#: word is then required to *mean* zero — which "0.01" does not. +_ALL_OTHER_RE = re.compile(r"all other off-diagonal rates (?:are )?([A-Za-z0-9.]+)") +_ZERO_WORDS = frozenset({"0", "0.0", "zero"}) #: ``kbright=7 s^-1`` inside the level-3 ``note``, keyed on the name. _BLINKING_RE = r"\b{name}\s*=\s*([0-9.]+)" @@ -740,7 +774,7 @@ def _registry() -> list[Pin]: at(KINSOFT, f"$.levels.{level}.n_traces"), ) ) - rates3 = "$.levels.level3.ground_truth.rates_s_inv" + rates3 = _LEVEL3_RATES_POINTER pins += [ Prose(_KINSOFT_SECTION, rates3, rate, f"{rates3}.{rate}", at(KINSOFT, f"{rates3}.{rate}")) for rate in _LEVEL3_RATES @@ -918,6 +952,32 @@ def test_the_page_quotes_the_artifact_verbatim( ) +def test_the_deferred_matrix_still_says_every_unlisted_rate_is_zero() -> None: + """The clause that makes six numbers a 4x4 matrix, on both sides. + + ``rates_s_inv`` lists only the six non-zero transitions; *"all other off-diagonal + rates zero"* is what completes the accepted ground truth. Pinning the six and not the + clause would let the page say the unlisted transitions are 0.01 with every other + guard green — and the deferred level-3 matrix is exactly what a future dwell-CDF + oracle would be built against. + """ + page = _flat(paragraph(PAGE_TEXT, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)) + note = str(at(KINSOFT, _LEVEL3_NOTE_POINTER)) + for label, text in (("the artifact note", note), ("docs/validation.md", page)): + match = _ALL_OTHER_RE.search(text) + assert match, f"{label} no longer states what the unlisted off-diagonal rates are" + stated = match.group(1).rstrip(".,;)") + assert stated in _ZERO_WORDS, ( + f"{label} says the unlisted off-diagonal rates are {stated!r}, not zero" + ) + listed = at(KINSOFT, _LEVEL3_RATES_POINTER) + assert isinstance(listed, dict) + assert set(listed) == set(_LEVEL3_RATES), ( + "the page enumerates a different rate set than the artifact records, so its " + '"all other" clause no longer covers what it claims to' + ) + + def test_only_one_method_was_measured_as_the_page_says() -> None: """The page says `$.coverage.measured_methods` records *only* that one method. @@ -1017,6 +1077,14 @@ def _mutate(section: str, old: str, new: str) -> str: "`kbright` 8 s⁻¹", "$.levels.level3.ground_truth.note kbright", ), + # The SECOND of two statements of one constant in a single paragraph. The first + # stays correct, so this fails only because every occurrence has to agree. + ( + "(d) Ranker held-out cross-validation", + "`DEFAULT_SHIP_BAR_PTS == 10.0`", + "`DEFAULT_SHIP_BAR_PTS == 11.0`", + "ml.prequential.DEFAULT_SHIP_BAR_PTS", + ), ], ) def test_a_deliberate_one_value_mismatch_fails( @@ -1076,6 +1144,16 @@ def test_swapping_the_two_dois_fails() -> None: assert not satisfied(swapped, pin), f"{pin.source} was accepted under the wrong label" +def test_a_nonzero_all_other_clause_fails() -> None: + """The zero clause is a claim, so changing it has to break something.""" + nonzero = PAGE_TEXT.replace( + "all other off-diagonal rates zero", "all other off-diagonal rates 0.01" + ) + match = _ALL_OTHER_RE.search(_flat(paragraph(nonzero, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER))) + assert match is not None + assert match.group(1).rstrip(".,;)") not in _ZERO_WORDS + + def test_a_stale_duplicate_section_fails_rather_than_being_shadowed() -> None: """A second copy of a section must not hide behind the canonical one. From 5e8927318f690fb9e74f7ccc294e8c0ed147089a Mon Sep 17 00:00:00 2001 From: bioedca <131809205+bioedca@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:22:11 -0500 Subject: [PATCH 5/7] test(docs): reject a contradiction in a cell or a quote, not only in prose The previous round taught the Prose branch that keeping the right number is not enough if a wrong one is published beside it. Cell and Quote were left on the old first-match rule, so the identical stale restatement passed or failed depending only on whether the page happened to print it in a table. Codex found all three. _states now requires every comparable number in the selected cell to agree, so "0.95 (was 0.93)" fails. Quote compiles its template into a pattern and checks EVERY occurrence rather than testing for one substring, so a paragraph that keeps "(5 % average)" and adds "(6 % average)" fails. The level-3 zero clause is read with finditer on both sides, so a second contradictory clause after a correct one is no longer invisible. The licence template gains one character of context. Rendering it as a bare "({})" made it a pattern that also fires on the citation's "(2022)" two clauses earlier, which under all-occurrence checking is a false failure rather than a false pass - but it is still the template being wrong about where its claim lives. Also narrows an over-broad claim rather than chasing it. The docstring said EXCLUSIONS enumerates everything deliberately unpinned; the page restates some pinned values a second time in running prose (RMS <= 0.5 px in (a), the rounded pooled-worst figures in (b)) and those sit outside every selected region. A page-wide numerical manifest is real scope that #214's approved criteria do not carry, so the claim now says what the module actually does and points at #435. --- tests/test_docs_validation_numbers.py | 133 +++++++++++++++++++++----- 1 file changed, 111 insertions(+), 22 deletions(-) diff --git a/tests/test_docs_validation_numbers.py b/tests/test_docs_validation_numbers.py index 3fbd9377..e0f32a54 100644 --- a/tests/test_docs_validation_numbers.py +++ b/tests/test_docs_validation_numbers.py @@ -39,10 +39,18 @@ guard is dependency-free: no ``tether`` import, no SciPy/h5py, and it runs unmarked on the plain 3-OS ``test`` matrix. -**What is deliberately not pinned** is enumerated in :data:`EXCLUSIONS` with a reason -for each, because an unexplained omission on a validation page is indistinguishable from -an oversight. In short: figures whose source is upstream git history, a historical -Actions run, or a prose record of a local run that produced no machine-readable artifact. +**Scope.** The pinned set is the one #214's accepted criteria enumerate: both frozen +tolerance blocks, both ``pooled_worst`` blocks, the kinSoft matrices, the measurement +counts, and every module constant the page prints a value for. Within that set, +:data:`EXCLUSIONS` records what is deliberately left unpinned and why, because on a +validation page an unexplained omission and an oversight look identical. + +This is **not** a page-wide numerical-coverage manifest. The page restates some pinned +values a second time in running prose — ``RMS ≤ 0.5 px`` in (a)'s enforcing-tests +paragraph, the rounded ``$.pooled_worst`` figures in (b)'s blockquote — and those +restatements sit outside the table row or anchored paragraph each pin selects, so they +are unguarded. Closing that is #435, deliberately kept separate from the scope #214's +title/body snapshot was approved against. """ from __future__ import annotations @@ -242,14 +250,15 @@ def _states(cell: str, value: object) -> bool: candidates = {cell.strip(), cell.strip().strip("`").strip('"')} candidates |= set(_BACKTICKED_RE.findall(cell)) | set(_QUOTED_RE.findall(cell)) return value in candidates + numbers: list[object] = [] for text in _NUMBER_RE.findall(cell): - try: - parsed = ast.literal_eval(text) - except (SyntaxError, ValueError): # pragma: no cover - defensive - continue - if type(parsed) is type(value) and parsed == value: - return True - return False + with contextlib.suppress(SyntaxError, ValueError): + numbers.append(ast.literal_eval(text)) + # Every number in the cell must agree, not merely one of them: `0.9 (also 0.8)` keeps + # the right value while publishing a contradiction, which is the same defect the + # Prose branch rejects. Every pinned Value cell states exactly one number today. + comparable = [x for x in numbers if _same_kind(x, value)] + return bool(comparable) and all(type(x) is type(value) and x == value for x in comparable) # --- The three pin kinds ------------------------------------------------------- @@ -306,10 +315,14 @@ class Quote(NamedTuple): def label(self) -> str: return f"{self.anchor}|{self.template}" - def rendered(self) -> str: + def value_text(self) -> str: + """The live value as the page spells it.""" if isinstance(self.value, str): - return self.template.format(self.value) - return self.template.format(f"{self.value * self.scale:g}") + return self.value + return f"{self.value * self.scale:g}" + + def rendered(self) -> str: + return self.template.format(self.value_text()) Pin = Cell | Prose | Quote @@ -375,6 +388,19 @@ def paragraph(text: str, section: str, anchor: str) -> str: return matches[0] +def _template_regex(template: str) -> re.Pattern[str]: + """``"Paper doi `{}`"`` as a pattern that captures whatever value is printed there. + + Used so a :class:`Quote` can check *every* place its claim is made rather than + stopping at the first. The placeholder captures a run of non-space characters, which + is what every pinned value is; the surrounding words are matched literally, so the + pattern only fires where the claim is actually being made. + """ + head, placeholder, tail = template.partition("{}") + assert placeholder, f"a Quote template needs a {{}} placeholder: {template!r}" + return re.compile(re.escape(head) + r"(\S+?)" + re.escape(tail)) + + def _flat(text: str) -> str: """One line, single-spaced — so a pinned phrase survives Markdown re-wrapping. @@ -416,7 +442,10 @@ def satisfied(text: str, pin: Pin) -> bool: return bool(comparable) and all( type(x) is type(pin.value) and x == pin.value for x in comparable ) - return _flat(pin.rendered()) in _flat(paragraph(text, pin.section, pin.anchor)) + # Every occurrence of the claim must carry the live value. A substring test would + # accept a paragraph that keeps `(5 % average)` and adds `(6 % average)` beside it. + stated = _template_regex(pin.template).findall(_flat(paragraph(text, pin.section, pin.anchor))) + return bool(stated) and all(found == pin.value_text() for found in stated) # --- The registry -------------------------------------------------------------- @@ -658,7 +687,10 @@ def _registry() -> list[Pin]: for key, template, value in ( ("doi", "Paper doi `{}`", at(KINSOFT, "$.source.doi")), ("data_doi", "data doi `{}`", at(KINSOFT, "$.source.data_doi")), - ("license", "({})", at(KINSOFT, "$.source.license")), + # The licence follows the data DOI's closing backtick. A bare "({})" would + # also fire on the "(2022)" of the citation two clauses earlier, so the + # template keeps just enough context to name the one place it belongs. + ("license", "` ({})", at(KINSOFT, "$.source.license")), ) ] @@ -823,6 +855,13 @@ def _registry() -> list[Pin]: "docs/adr/0022 as prose. There is no machine-readable artifact, so a pin would " "only assert that two transcriptions of the same paragraph agree." ), + "second restatements of a pinned value elsewhere in the page's prose": ( + "Each pin selects one table row or one anchored paragraph, so a value the page " + "also states somewhere else -- `RMS <= 0.5 px` in (a)'s enforcing-tests " + "paragraph, the rounded pooled-worst figures in (b)'s blockquote -- is outside " + "every selected region. A page-wide numerical manifest is real work and real " + "scope, and #214's criteria enumerate a specific set; tracked in #435." + ), "the paper's '14 published analyses', '17 %' and '9-14 %' figures": ( "Facts of Gotz et al. 2022. kinsoft_reference.json records them only inside the " "free-text `spread_source` and `deferred_reason` strings, which the page " @@ -964,12 +1003,12 @@ def test_the_deferred_matrix_still_says_every_unlisted_rate_is_zero() -> None: page = _flat(paragraph(PAGE_TEXT, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)) note = str(at(KINSOFT, _LEVEL3_NOTE_POINTER)) for label, text in (("the artifact note", note), ("docs/validation.md", page)): - match = _ALL_OTHER_RE.search(text) - assert match, f"{label} no longer states what the unlisted off-diagonal rates are" - stated = match.group(1).rstrip(".,;)") - assert stated in _ZERO_WORDS, ( - f"{label} says the unlisted off-diagonal rates are {stated!r}, not zero" - ) + # Every occurrence, not the first: a second, contradictory clause added after a + # correct one would otherwise never be looked at. + clauses = [m.group(1).rstrip(".,;)") for m in _ALL_OTHER_RE.finditer(text)] + assert clauses, f"{label} no longer states what the unlisted off-diagonal rates are" + wrong = [c for c in clauses if c not in _ZERO_WORDS] + assert not wrong, f"{label} says the unlisted off-diagonal rates are {wrong}, not zero" listed = at(KINSOFT, _LEVEL3_RATES_POINTER) assert isinstance(listed, dict) assert set(listed) == set(_LEVEL3_RATES), ( @@ -1144,6 +1183,56 @@ def test_swapping_the_two_dois_fails() -> None: assert not satisfied(swapped, pin), f"{pin.source} was accepted under the wrong label" +@pytest.mark.parametrize( + ("section", "old", "new", "source"), + [ + # A table cell that keeps the right value and publishes a stale one beside it. + ( + _PARITY_SECTION, + "| `viterbi_min_agreement` | floor | 0.95 |", + "| `viterbi_min_agreement` | floor | 0.95 (was 0.93) |", + "$.tolerance.viterbi_min_agreement", + ), + # The same shape in a Quote's paragraph rather than a cell. + ( + _KINSOFT_SECTION, + "(5 % average)", + "(5 % average), or (6 % average)", + "$.levels.level1.reported_inter_tool_spread.rate_mean_rel_dev_from_gt", + ), + ], +) +def test_a_contradiction_beside_a_correct_value_fails( + section: str, old: str, new: str, source: str +) -> None: + """Keeping the right number is not enough if a wrong one is published beside it. + + Cell, Prose and Quote all have to agree about this. They did not: only the Prose + branch rejected a contradiction, so the same stale restatement passed or failed + depending on whether the page happened to print it in a table. + """ + mutated = _mutate(section, old, new) + pins = [p for p in REGISTRY if p.source == source] + assert pins, f"no registry entry sources {source}" + for pin in pins: + assert satisfied(PAGE_TEXT, pin), f"{source} does not hold on the unmutated page" + assert not satisfied(mutated, pin), ( + f"{source} was accepted while the page also stated a contradicting value" + ) + + +def test_a_second_contradictory_zero_clause_fails() -> None: + """The zero clause is checked everywhere it appears, not just the first time.""" + doubled = PAGE_TEXT.replace( + "all other off-diagonal rates zero", + "all other off-diagonal rates zero, and all other off-diagonal rates 0.01", + ) + text = _flat(paragraph(doubled, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)) + clauses = [m.group(1).rstrip(".,;)") for m in _ALL_OTHER_RE.finditer(text)] + assert len(clauses) == 2, clauses + assert [c for c in clauses if c not in _ZERO_WORDS] == ["0.01"] + + def test_a_nonzero_all_other_clause_fails() -> None: """The zero clause is a claim, so changing it has to break something.""" nonzero = PAGE_TEXT.replace( From 6377f4636debda3f86ae7bf677fa71c1ec4c8248 Mon Sep 17 00:00:00 2001 From: bioedca <131809205+bioedca@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:02:40 -0500 Subject: [PATCH 6/7] test(docs): finish the contradiction rule and make the mutation tests bite Fifth Codex read. Five findings, all in the class the previous rounds were closing, so they are finished here rather than left half-done. The string branch of _states was still first-match while the numeric branch required agreement, so a date or interpreter-version cell could keep the right token and publish a stale one beside it. A cell now has to state exactly one value. That completes the rule the docstring already claims: Cell, Prose and Quote all reject a contradiction, not only a wrong value. _blinking_rate read the note with [0-9.]+, which truncates kbright=7e-1 to 7 and would let the page keep saying 7 while the artifact had come to mean 0.7. The literal is matched whole - sign, decimals, exponent, trailing boundary - and every occurrence must agree, so a note that contradicts itself has no single value to pin against and says so. The completeness test filtered constants by splitting a code span on whitespace, so a conventional `NEW_LIMIT=5` produced the token "NEW_LIMIT=5" while the literal scanner produced "NEW_LIMIT". The two never met and a new page-printed constant could arrive unpinned. It now takes the leading identifier. Three mutation tests re-implemented the check they were meant to protect, so they would have kept passing after the real guard regressed - a test that tests itself. The zero-clause scan and the two-sided quotation check are now functions the guard and the mutation tests both call. --- tests/test_docs_validation_numbers.py | 139 ++++++++++++++++++-------- 1 file changed, 97 insertions(+), 42 deletions(-) diff --git a/tests/test_docs_validation_numbers.py b/tests/test_docs_validation_numbers.py index e0f32a54..cc1f89a1 100644 --- a/tests/test_docs_validation_numbers.py +++ b/tests/test_docs_validation_numbers.py @@ -125,6 +125,8 @@ def module_constant(dotted: str, name: str) -> object: #: sit next to a number without being one; the page only ever writes real code in code #: spans. _CONSTANT_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") +#: The identifier a code span opens with, however it continues (`X = 1`, `X=1`, `X`). +_LEADING_IDENT_RE = re.compile(r"^[A-Za-z_]\w*") #: Name / string / number, tried in that order so ``smd_281mol`` and ``k12`` stay whole #: names rather than decomposing into a name and a stray number. Slashes and hyphens are @@ -247,9 +249,14 @@ def _states(cell: str, value: object) -> bool: required to match, so ``0`` never satisfies ``0.0``. """ if isinstance(value, str): - candidates = {cell.strip(), cell.strip().strip("`").strip('"')} - candidates |= set(_BACKTICKED_RE.findall(cell)) | set(_QUOTED_RE.findall(cell)) - return value in candidates + # Explicit transcriptions first -- code spans and quoted strings -- and the bare + # cell only when it carries neither, because the page writes `2026-06-26` bare in + # a table and ``"2026-06-26"`` in prose. The cell must state exactly ONE of them: + # keeping the right date and adding a stale one beside it is a contradiction, the + # same way it is for a number. A cell that legitimately needs a second code span + # fails here loudly rather than quietly widening what counts. + tokens = set(_BACKTICKED_RE.findall(cell)) | set(_QUOTED_RE.findall(cell)) + return (tokens or {cell.strip()}) == {value} numbers: list[object] = [] for text in _NUMBER_RE.findall(cell): with contextlib.suppress(SyntaxError, ValueError): @@ -477,8 +484,36 @@ def satisfied(text: str, pin: Pin) -> bool: _ALL_OTHER_RE = re.compile(r"all other off-diagonal rates (?:are )?([A-Za-z0-9.]+)") _ZERO_WORDS = frozenset({"0", "0.0", "zero"}) -#: ``kbright=7 s^-1`` inside the level-3 ``note``, keyed on the name. -_BLINKING_RE = r"\b{name}\s*=\s*([0-9.]+)" + +def zero_clauses(text: str) -> tuple[list[str], list[str]]: + """Every ``all other off-diagonal rates X`` clause in ``text``, split by whether X is zero. + + The guard *and* the tests that prove the guard bites both call this, so a regression + here fails both. Written as a helper for exactly that reason: a mutation test that + re-implements the check it is meant to protect keeps passing after the real check + regresses, which is a test that tests itself. + """ + stated = [m.group(1).rstrip(".,;)") for m in _ALL_OTHER_RE.finditer(text)] + return [c for c in stated if c in _ZERO_WORDS], [c for c in stated if c not in _ZERO_WORDS] + + +def quotation_sides(fragment: str, artifact: dict, pointer: str, region: str) -> tuple[bool, bool]: + """Is ``fragment`` still in the artifact, and still in the page region that quotes it? + + Shared by the live check and by the test that proves a dropped quotation fails, so + neither can drift away from the other. Returned as two flags so the caller can say + *which* side went stale. + """ + held = at(artifact, pointer) + source = held if isinstance(held, str) else "\n".join(str(item) for item in held) # type: ignore[union-attr] + return fragment in source, fragment in _flat(region) + + +#: ``kbright=7 s^-1`` inside the level-3 ``note``, keyed on the name. The literal is +#: matched whole — sign, decimals and exponent — with a trailing boundary, because +#: ``[0-9.]+`` would read ``kbright=7e-1`` as ``7`` and let the page keep saying 7 while +#: the artifact had come to mean 0.7. +_BLINKING_RE = r"\b{name}\s*=\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)(?![\w.])" def _blinking_rate(name: str) -> object: @@ -491,9 +526,13 @@ def _blinking_rate(name: str) -> object: nothing is prose. """ note = str(at(KINSOFT, "$.levels.level3.ground_truth.note")) - match = re.search(_BLINKING_RE.format(name=name), note) - assert match, f"the level-3 note no longer records `{name}=`; re-read it before pinning" - return ast.literal_eval(match.group(1)) + stated = {m.group(1) for m in re.finditer(_BLINKING_RE.format(name=name), note)} + assert stated, f"the level-3 note no longer records `{name}=`; re-read it before pinning" + assert len(stated) == 1, ( + f"the level-3 note states {name} as {sorted(stated)} — the artifact contradicts " + "itself, so there is no single value to pin the page to" + ) + return ast.literal_eval(stated.pop()) def _registry() -> list[Pin]: @@ -934,8 +973,14 @@ def test_every_constant_the_page_states_a_value_for_is_pinned() -> None: """ pinned = {p.key for p in REGISTRY if isinstance(p, Prose)} pinned |= {p.row for p in REGISTRY if isinstance(p, Cell)} - # The leading token of each code span, so `DEFAULT_SHIP_BAR_PTS = 10.0` counts. - code_spans = {span.split()[0] for span in _BACKTICKED_RE.findall(PAGE_TEXT) if span.split()} + # The leading *identifier* of each code span, so `DEFAULT_SHIP_BAR_PTS = 10.0` counts + # and so does `NEW_LIMIT=5`, which splitting on whitespace would have left as the + # unmatchable token "NEW_LIMIT=5" while `_owned_literals` called it "NEW_LIMIT". + code_spans = { + match.group() + for span in _BACKTICKED_RE.findall(PAGE_TEXT) + if (match := _LEADING_IDENT_RE.match(span.strip())) + } regions: list[str] = [] for body in _sections(PAGE_TEXT).values(): regions += _prose_paragraphs(body) @@ -979,13 +1024,14 @@ def test_the_page_quotes_the_artifact_verbatim( fragment: str, artifact: dict, pointer: str, section: str, anchor: str ) -> None: """A quotation has two sides, and the page owns one of them.""" - held = at(artifact, pointer) - source = held if isinstance(held, str) else "\n".join(str(item) for item in held) # type: ignore[union-attr] - assert fragment in source, ( + in_artifact, on_page = quotation_sides( + fragment, artifact, pointer, paragraph(PAGE_TEXT, section, anchor) + ) + assert in_artifact, ( f"docs/validation.md presents {fragment!r} as a verbatim quote of {pointer}, " "but the artifact no longer says it" ) - assert fragment in _flat(paragraph(PAGE_TEXT, section, anchor)), ( + assert on_page, ( f"{pointer} still says {fragment!r} but docs/validation.md no longer quotes it " "where it claims to" ) @@ -1003,12 +1049,9 @@ def test_the_deferred_matrix_still_says_every_unlisted_rate_is_zero() -> None: page = _flat(paragraph(PAGE_TEXT, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)) note = str(at(KINSOFT, _LEVEL3_NOTE_POINTER)) for label, text in (("the artifact note", note), ("docs/validation.md", page)): - # Every occurrence, not the first: a second, contradictory clause added after a - # correct one would otherwise never be looked at. - clauses = [m.group(1).rstrip(".,;)") for m in _ALL_OTHER_RE.finditer(text)] - assert clauses, f"{label} no longer states what the unlisted off-diagonal rates are" - wrong = [c for c in clauses if c not in _ZERO_WORDS] - assert not wrong, f"{label} says the unlisted off-diagonal rates are {wrong}, not zero" + zero, nonzero = zero_clauses(text) + assert zero or nonzero, f"{label} no longer states what the unlisted off-diagonal rates are" + assert not nonzero, f"{label} says the unlisted off-diagonal rates are {nonzero}, not zero" listed = at(KINSOFT, _LEVEL3_RATES_POINTER) assert isinstance(listed, dict) assert set(listed) == set(_LEVEL3_RATES), ( @@ -1221,26 +1264,31 @@ def test_a_contradiction_beside_a_correct_value_fails( ) -def test_a_second_contradictory_zero_clause_fails() -> None: - """The zero clause is checked everywhere it appears, not just the first time.""" - doubled = PAGE_TEXT.replace( - "all other off-diagonal rates zero", - "all other off-diagonal rates zero, and all other off-diagonal rates 0.01", - ) - text = _flat(paragraph(doubled, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)) - clauses = [m.group(1).rstrip(".,;)") for m in _ALL_OTHER_RE.finditer(text)] - assert len(clauses) == 2, clauses - assert [c for c in clauses if c not in _ZERO_WORDS] == ["0.01"] - +@pytest.mark.parametrize( + ("label", "replacement", "expected_nonzero"), + [ + # The clause changed outright. + ("changed", "all other off-diagonal rates 0.01", ["0.01"]), + # A correct clause kept, a contradicting one added after it. Reading only the + # first occurrence would call this page clean. + ( + "contradicted", + "all other off-diagonal rates zero, and all other off-diagonal rates 0.01", + ["0.01"], + ), + ], +) +def test_a_bad_zero_clause_fails(label: str, replacement: str, expected_nonzero: list[str]) -> None: + """Both failures go through :func:`zero_clauses`, the function the guard itself calls. -def test_a_nonzero_all_other_clause_fails() -> None: - """The zero clause is a claim, so changing it has to break something.""" - nonzero = PAGE_TEXT.replace( - "all other off-diagonal rates zero", "all other off-diagonal rates 0.01" - ) - match = _ALL_OTHER_RE.search(_flat(paragraph(nonzero, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER))) - assert match is not None - assert match.group(1).rstrip(".,;)") not in _ZERO_WORDS + Re-implementing the scan here would make this test pass even after the guard regressed + to reading one occurrence — a mutation test that only proves the mutation happened. + """ + mutated = PAGE_TEXT.replace("all other off-diagonal rates zero", replacement) + page = _flat(paragraph(mutated, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)) + assert zero_clauses(page)[1] == expected_nonzero, label + # And the unmutated page is clean by the same function. + assert not zero_clauses(_flat(paragraph(PAGE_TEXT, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)))[1] def test_a_stale_duplicate_section_fails_rather_than_being_shadowed() -> None: @@ -1260,11 +1308,18 @@ def test_a_stale_duplicate_section_fails_rather_than_being_shadowed() -> None: def test_a_dropped_quotation_fails_even_though_the_artifact_still_says_it() -> None: - """The page owns one side of every quotation it presents as verbatim.""" + """The page owns one side of every quotation it presents as verbatim. + + Goes through :func:`quotation_sides`, the same function the live check calls, so a + regression that stopped consulting the page would fail here too. + """ fragment, artifact, pointer, section, anchor = _VERBATIM_QUOTES[0] - assert fragment in str(at(artifact, pointer)) # the artifact side is untouched reworded = PAGE_TEXT.replace(f'it "{fragment}"', "it says otherwise") - assert fragment not in _flat(paragraph(reworded, section, anchor)) + before = quotation_sides(fragment, artifact, pointer, paragraph(PAGE_TEXT, section, anchor)) + after = quotation_sides(fragment, artifact, pointer, paragraph(reworded, section, anchor)) + assert before == (True, True) + # The artifact side is untouched; only the page stopped quoting it. + assert after == (True, False) def test_an_ambiguous_anchor_fails_loudly_rather_than_picking_one() -> None: From ed892b6f073875970293f10b5295206035f29cad Mon Sep 17 00:00:00 2001 From: bioedca <131809205+bioedca@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:15:50 -0500 Subject: [PATCH 7/7] test(docs): make column and row selection as strict as the rest Three structural-selection holes, each one a place the module already claimed to assert a unique match and did not. A duplicate target column bound every pin to the first one, because `in` followed by `.index()` never counts. A stale second `$.tolerance` column could therefore publish a contradicting value with the guard green. The count is now asserted. A bare row label matched by prefix with no boundary, so `level2` still answered for a row renamed or mistyped `level20 (Fig. 3)` and validated that row's numbers under a label the page no longer uses. Uniqueness could not catch it on its own: only one row still started with `level2`. A prefix match now has to end at a non-identifier character. The level-3 rate-set assertion read the artifact twice and the page never, so its own failure message described a comparison it was not making. The page could add `k13 0.01` beside the six correct rates and still claim every other transition was zero - a contradiction with itself rather than with the JSON. The transitions the page enumerates are now extracted and compared. --- tests/test_docs_validation_numbers.py | 98 ++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/tests/test_docs_validation_numbers.py b/tests/test_docs_validation_numbers.py index cc1f89a1..984171d6 100644 --- a/tests/test_docs_validation_numbers.py +++ b/tests/test_docs_validation_numbers.py @@ -353,19 +353,34 @@ def _table(text: str, pin: Cell) -> Table: return candidates[0] +def _row_matches(cell: str, label: str) -> bool: + """Is ``cell`` the row labelled ``label``? + + A backticked token has to match exactly. A bare label may be a prefix — the deferred + levels are written ``level2 (Fig. 3)`` — but only up to a boundary, so ``level2`` no + longer answers for a row renamed or mistyped as ``level20``, which would quietly + validate that row's numbers under the wrong label instead of failing selection. + """ + if label in _BACKTICKED_RE.findall(cell): + return True + if not cell.startswith(label): + return False + rest = cell[len(label) :] + return not rest or not (rest[0].isalnum() or rest[0] == "_") + + def cell_text(text: str, pin: Cell) -> str: """The single cell a :class:`Cell` pin addresses.""" table = _table(text, pin) - assert pin.column in table.header, ( - f"the {pin.table!r} table in '{pin.section}' has no {pin.column!r} column; " - f"its header is {list(table.header)}" + # Exactly one column, not merely at least one: a stale duplicate `$.tolerance` column + # would bind every pin to the first and leave the contradicting one published. + found = table.header.count(pin.column) + assert found == 1, ( + f"the {pin.table!r} table in '{pin.section}' must have exactly one {pin.column!r} " + f"column; found {found}. Its header is {list(table.header)}" ) column = table.header.index(pin.column) - matches = [ - row - for row in table.rows - if pin.row in _BACKTICKED_RE.findall(row[0]) or row[0].startswith(pin.row) - ] + matches = [row for row in table.rows if _row_matches(row[0], pin.row)] assert len(matches) == 1, ( f"the {pin.table!r} table in '{pin.section}' must hold exactly one row for " f"{pin.row!r}; found {len(matches)}" @@ -481,6 +496,8 @@ def satisfied(text: str, pin: Pin) -> bool: #: The clause that turns six numbers into a 4x4 matrix, on either side. The artifact #: writes "are 0" and the page writes "zero", so it is matched by shape and the captured #: word is then required to *mean* zero — which "0.01" does not. +#: A transition the page enumerates: ``k12``, never ``kbright``. +_TRANSITION_RE = re.compile(r"^k\d+$") _ALL_OTHER_RE = re.compile(r"all other off-diagonal rates (?:are )?([A-Za-z0-9.]+)") _ZERO_WORDS = frozenset({"0", "0.0", "zero"}) @@ -1055,8 +1072,23 @@ def test_the_deferred_matrix_still_says_every_unlisted_rate_is_zero() -> None: listed = at(KINSOFT, _LEVEL3_RATES_POINTER) assert isinstance(listed, dict) assert set(listed) == set(_LEVEL3_RATES), ( - "the page enumerates a different rate set than the artifact records, so its " - '"all other" clause no longer covers what it claims to' + "the artifact records a different rate set than this module enumerates; re-read " + "the level-3 ground truth before pinning it" + ) + # And the same set has to be what the PAGE prints. Reading only the artifact here + # would let the page add `k13 0.01` beside the six correct rates while still saying + # every other transition is zero -- a contradiction with itself, not with the JSON. + displayed = { + name + for name, values in _owned_literals( + paragraph(PAGE_TEXT, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER) + ).items() + if _TRANSITION_RE.match(name) and values + } + assert displayed == set(listed), ( + f"docs/validation.md enumerates transitions {sorted(displayed)} but the artifact " + f'records {sorted(listed)}, so its "all other ... zero" clause no longer covers ' + "what it claims to" ) @@ -1291,6 +1323,52 @@ def test_a_bad_zero_clause_fails(label: str, replacement: str, expected_nonzero: assert not zero_clauses(_flat(paragraph(PAGE_TEXT, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER)))[1] +def test_a_duplicate_column_fails_rather_than_binding_to_the_first() -> None: + """A stale second column must not hide behind the canonical one. + + ``header.index()`` always returns the first, so without a count the pins would keep + reading the good column while the contradicting one stayed published. + """ + pin = next(p for p in REGISTRY if isinstance(p, Cell) and p.column == "`$.tolerance` (default)") + duplicated = PAGE_TEXT.replace( + "| Bound | Direction | `$.tolerance` (default) | `$.tolerance_by_method.ebhmm` |", + "| Bound | Direction | `$.tolerance` (default) | `$.tolerance` (default) |" + " `$.tolerance_by_method.ebhmm` |", + ) + with pytest.raises(AssertionError, match="exactly one"): + cell_text(duplicated, pin) + + +def test_a_row_label_only_matches_at_a_boundary() -> None: + """`level2` must not answer for a row renamed `level20`. + + The uniqueness assertion cannot catch this on its own — only one row would still + start with `level2` — so the pin would validate that row's numbers under a label the + page no longer uses. + """ + assert _row_matches("level2 (Fig. 3)", "level2") + assert _row_matches("level2", "level2") + assert not _row_matches("level20 (Fig. 3)", "level2") + assert not _row_matches("level2_old", "level2") + # A backticked label still has to match the whole token. + assert _row_matches("`k12_low_high`", "k12_low_high") + assert not _row_matches("`k12_low_high_stale`", "k12_low_high") + + +def test_an_extra_transition_on_the_page_fails() -> None: + """A seventh rate printed beside the six contradicts the "all other zero" clause.""" + extra = PAGE_TEXT.replace("k32 0.680", "k13 0.010, k32 0.680") + displayed = { + name + for name, values in _owned_literals( + paragraph(extra, _KINSOFT_SECTION, _LEVEL3_RATES_POINTER) + ).items() + if _TRANSITION_RE.match(name) and values + } + assert "k13" in displayed + assert displayed != set(at(KINSOFT, _LEVEL3_RATES_POINTER)) + + def test_a_stale_duplicate_section_fails_rather_than_being_shadowed() -> None: """A second copy of a section must not hide behind the canonical one.