From a7e7a7b42f4f9aa28f7f3525c32377a142addd06 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Mon, 7 Sep 2026 16:06:21 -0400 Subject: [PATCH 1/3] fix: validate metadata-only release readiness evidence --- AGENTS.md | 4 + doc/high_risk_pre_merge_readiness.md | 49 +- .../tooling/high_risk_readiness_test.dart | 469 ++++++++++++++++++ tool/testing/classify_high_risk_changes.dart | 1 + tool/testing/high_risk_readiness.dart | 132 ++++- .../high_risk_readiness_evidence.schema.json | 10 +- tool/testing/release_metadata_readiness.dart | 282 +++++++++++ tool/testing/test_matrix.dart | 13 + 8 files changed, 953 insertions(+), 7 deletions(-) create mode 100644 tool/testing/release_metadata_readiness.dart diff --git a/AGENTS.md b/AGENTS.md index 34a24c1d..8491e942 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -334,6 +334,10 @@ require issue-specific positive and negative tests that fail if the relevant branch is deleted, bypassed, or miswired. Record zero known PR-caused P1 regressions and zero unresolved review threads in the PR's high-risk block and evidence payload. +For core-patch release metadata only, use the bounded +`release-metadata-verification` evidence route described in +`doc/high_risk_pre_merge_readiness.md`. It checks exact Git blobs and existing +release tests; do not fabricate changed tests or reclassify the PR as standard. Structured-output changes must cover compiled grammar acceptance and rejection, schema-directed scalar and container reconstruction, partial-streaming diff --git a/doc/high_risk_pre_merge_readiness.md b/doc/high_risk_pre_merge_readiness.md index 03385372..29ac5c56 100644 --- a/doc/high_risk_pre_merge_readiness.md +++ b/doc/high_risk_pre_merge_readiness.md @@ -43,7 +43,7 @@ The contract fails closed against: 3. duplicate JSON keys, unknown fields, wrong scalar/container types, and caller-declared decisions; 4. caller-forged changed-file lists; -5. deleted, renamed-old, unchanged, absolute, traversal, wildcard, non-test, or +5. deleted, renamed-old, unauthorized unchanged, absolute, traversal, wildcard, non-test, or phantom evidence paths; 6. boolean-only structured-output attestations without named production tests; 7. PR-authored workflow or evidence execution; @@ -93,7 +93,52 @@ source and destination paths are both classified. Every cited test must: This intentionally rejects unchanged tests as issue-specific proof. Existing coverage can still inform a human audit, but it cannot satisfy the changed -production-test evidence field. +production-test evidence field except through the bounded release route below. + +### Core-patch release metadata evidence + +A release-only PR remains **high-risk / artifactConsumer**. It can additionally +declare the `release-metadata-verification` matrix row, with this exact command +and a `pass` result (never `notApplicable`): + +```bash +dart run tool/testing/verify_release_docs_versions.dart --release-prep && dart test -p vm -j 1 test/unit/tooling/verify_release_docs_companion_pins_test.dart +``` + +For this route, `affected_test_paths` must name exactly that existing test. +The independent audit and both matrix rows remain required and are bound to +the exact head and base. This records an existing release regression suite, +not a fabricated newly changed test. The evaluator does not execute candidate +scripts and cannot authenticate a claimed test run; the independent reviewer +must inspect the exact-head command/log evidence. + +Eligibility comes from literal Git blobs and the complete rename-aware diff, +not a `metadata_only` flag or a caller-provided inventory: + +- `pubspec.yaml` changes only its canonical stable version to the next patch; + every other byte, including dependencies, SDK, hooks and overrides, is fixed. +- Both current changelogs and the four maintained installation README/docs + pages are modified. Historical numbered changelog sections, runtime identity + strings, embedded MDX/HTML fragments, frontmatter and already-prepared + companion constraints remain unchanged; current core snippets name the new + patch. Narrative release prose still needs independent review. +- The only optional path is `example/chat_app/pubspec.lock`, changing only the + matching local `llamadart` version. Path, source, inventory, hashes, SDK and + all other bytes remain identical. Pub must generate the lock normally. +- All changed paths are existing non-executable regular files; additions, + deletions, copies, renames and mode changes are excluded. +- The existing release verifier and named regression test are unchanged + regular blobs at both revisions. + +The exact allowlist lives in `tool/testing/release_metadata_readiness.dart`. +Companion version changes, runtime pins, generated bindings, hooks, SwiftPM, +workflows, security/review policy, other docs/MDX and production changes cannot +use this exception. Broader release changes use the ordinary changed-test +contract instead of expanding this route implicitly. + +An internally consistent release evaluation still returns +`unverifiedPrerequisites` (exit 2), never operational readiness. External +authentication/publication/ruleset boundaries below are unchanged. ### Independent audit diff --git a/test/unit/tooling/high_risk_readiness_test.dart b/test/unit/tooling/high_risk_readiness_test.dart index 8f770cda..fe555ce2 100644 --- a/test/unit/tooling/high_risk_readiness_test.dart +++ b/test/unit/tooling/high_risk_readiness_test.dart @@ -8,6 +8,7 @@ import 'package:test/test.dart'; import '../../../tool/testing/classify_high_risk_changes.dart'; import '../../../tool/testing/high_risk_readiness.dart'; +import '../../../tool/testing/release_metadata_readiness.dart'; const baseSha = '1111111111111111111111111111111111111111'; const headSha = '2222222222222222222222222222222222222222'; @@ -30,6 +31,7 @@ class FakeRepositoryState implements RepositoryStateReader { this.commitsExist = true, this.baseIsAncestor = true, this.pathProbeThrows = false, + this.files = const {}, }) : existingPaths = existingPaths ?? changes @@ -42,8 +44,16 @@ class FakeRepositoryState implements RepositoryStateReader { final bool commitsExist; final bool baseIsAncestor; final bool pathProbeThrows; + final Map> files; var repositoryCallCount = 0; + @override + Future fileAt( + String sha, + String path, { + String? workingDirectory, + }) async => files[sha]?[path]; + @override Future commitExists(String sha, {String? workingDirectory}) async { repositoryCallCount++; @@ -160,6 +170,69 @@ Map backendEvidence() { return evidence; } +Map metadataEvidence() { + final evidence = backendEvidence(); + evidence['surfaces'] = ['artifactConsumer']; + (evidence['required_matrix_row_ids'] as List).add(releaseMetadataRow); + (evidence['matrix_row_evidence'] as Map)[releaseMetadataRow] = { + 'row_id': releaseMetadataRow, + 'result': 'pass', + 'command': releaseMetadataCommand, + 'evidence_notes': + 'Strict verifier and existing companion-pin tests passed on exact head.', + }; + evidence['affected_test_paths'] = [releaseMetadataTest]; + return evidence; +} + +Map> metadataFiles() { + ReadinessFile file(String text) => (mode: '100644', contents: text); + final base = { + 'pubspec.yaml': file( + 'name: llamadart\nversion: 0.8.22\ndependencies:\n yaml: ^3.1.3\n', + ), + for (final path in releaseMetadataDocs) + path: file( + path.contains('CHANGELOG') || path.contains('recent-releases') + ? '# Changes\n\n## Unreleased\n\n* Update.\n\n## 0.8.22\n\n* History.\n' + : '# Install\n\n```yaml\ndependencies:\n llamadart: ^0.8.22\n```\n', + ), + releaseMetadataLock: file( + 'packages:\n llamadart:\n dependency: "direct main"\n description:\n path: "../.."\n relative: true\n source: path\n version: "0.8.22"\nsdks:\n dart: ">=3.13.0 <4.0.0"\n', + ), + releaseMetadataVerifier: file('// maintained verifier\n'), + releaseMetadataTest: file( + '// existing negative companion pin regressions\n', + ), + }; + final head = Map.from(base); + for (final path in releaseMetadataPaths) { + final old = base[path]!; + head[path] = file( + path.contains('CHANGELOG') || path.contains('recent-releases') + ? old.contents.replaceFirst('## Unreleased', '## 0.8.23') + : old.contents.replaceAll('0.8.22', '0.8.23'), + ); + } + return {baseSha: base, headSha: head}; +} + +Future evaluateMetadata({ + Map? evidence, + Map>? files, + List? changes, +}) => HighRiskReadinessEvaluator( + repositoryState: FakeRepositoryState( + changes: + changes ?? + [ + for (final path in releaseMetadataPaths) + RepositoryChange(path: path, kind: RepositoryChangeKind.modified), + ], + files: files ?? metadataFiles(), + ), +).evaluate(evidence: evidence ?? metadataEvidence(), context: context); + Map standardEvidence() { final evidence = backendEvidence(); evidence['classification'] = 'standard'; @@ -262,6 +335,401 @@ void mutateIdentityToOtherValidValues(Map evidence) { } void main() { + group('bounded metadata-only release evidence', () { + test( + 'real Git blobs authorize only the committed metadata candidate', + () async { + final repo = Directory.systemTemp.createTempSync( + 'release-metadata-git-', + ); + addTearDown(() => repo.deleteSync(recursive: true)); + String git(List args) { + final result = Process.runSync( + 'git', + args, + workingDirectory: repo.path, + ); + expect(result.exitCode, 0, reason: '${result.stderr}'); + return (result.stdout as String).trim(); + } + + git(['init', '--quiet']); + git(['config', 'user.name', 'Metadata test']); + git(['config', 'user.email', 'metadata@example.invalid']); + git(['config', 'core.autocrlf', 'false']); + final files = metadataFiles(); + void write(Map tree) { + for (final entry in tree.entries) { + File('${repo.path}/${entry.key}') + ..parent.createSync(recursive: true) + ..writeAsStringSync(entry.value.contents); + } + } + + write(files[baseSha]!); + git(['add', '.']); + git(['commit', '--quiet', '-m', 'base']); + final base = git(['rev-parse', 'HEAD']); + write(files[headSha]!); + git(['add', '.']); + git(['commit', '--quiet', '-m', 'release']); + final head = git(['rev-parse', 'HEAD']); + final evidence = metadataEvidence() + ..['expected_pr_head_sha'] = head + ..['current_base_sha'] = base; + evidence['independent_audit']['audit_head_sha'] = head; + evidence['independent_audit']['audit_base_sha'] = base; + Future evaluate( + String candidate, + Map input, + ) => const HighRiskReadinessEvaluator().evaluate( + evidence: input, + context: PullRequestContext( + repository: context.repository, + prNumber: context.prNumber, + headSha: candidate, + baseSha: base, + author: context.author, + ), + workingDirectory: repo.path, + ); + // Mutable working-tree content cannot substitute for committed blobs. + File('${repo.path}/README.md').writeAsStringSync('{unsafe()}'); + expect( + (await evaluate(head, evidence)).decision, + ReadinessDecision.unverifiedPrerequisites, + ); + git(['add', 'README.md']); + git(['commit', '--quiet', '-m', 'unsafe MDX']); + final unsafe = git(['rev-parse', 'HEAD']); + expectFailure( + await evaluate(unsafe, evidence), + ReadinessFailureClassification.headMismatch, + ); + evidence['expected_pr_head_sha'] = unsafe; + evidence['independent_audit']['audit_head_sha'] = unsafe; + expectFailure( + await evaluate(unsafe, evidence), + ReadinessFailureClassification.invalidReleaseMetadata, + ); + }, + ); + test( + 'documented grammar and repeated existing runtime identity are inert', + () async { + final files = metadataFiles(); + for (final sha in [baseSha, headSha]) { + final old = files[sha]!['README.md']!; + files[sha]!['README.md'] = ( + mode: old.mode, + contents: '${old.contents}\nNative v0.4.0.\n', + ); + } + final candidate = files[headSha]!['README.md']!; + files[headSha]!['README.md'] = ( + mode: candidate.mode, + contents: + '${candidate.contents}\nKnown v0.4.0 limitation: `root ::= "a"{2000}`.\n', + ); + expect( + (await evaluateMetadata(files: files)).decision, + ReadinessDecision.unverifiedPrerequisites, + ); + }, + ); + test( + 'fence and inline context cannot convert existing fragments to MDX', + () async { + for (final snippet in [ + '{dangerous()}', + 'export const bad = 1;', + '', + after: '', + ), + ( + before: '', + after: '', + ), + ]) { + final files = metadataFiles(); + final base = files[baseSha]!['README.md']!; + final head = files[headSha]!['README.md']!; + files[baseSha]!['README.md'] = ( + mode: base.mode, + contents: '${base.contents}\n${pair.before}\n', + ); + files[headSha]!['README.md'] = ( + mode: head.mode, + contents: '${head.contents}\n${pair.after}\n', + ); + expectFailure( + await evaluateMetadata(files: files), + ReadinessFailureClassification.invalidReleaseMetadata, + ); + } + }, + ); + test( + 'omitting a required lock or current document cannot hide stale metadata', + () async { + for (final omitted in releaseMetadataPaths) { + final changes = [ + for (final path in releaseMetadataPaths) + if (path != omitted) + RepositoryChange( + path: path, + kind: RepositoryChangeKind.modified, + ), + ]; + expectFailure( + await evaluateMetadata(changes: changes), + ReadinessFailureClassification.invalidReleaseMetadata, + ); + } + }, + ); test( 'real Git blobs authorize only the committed metadata candidate', () async { diff --git a/tool/testing/release_metadata_readiness.dart b/tool/testing/release_metadata_readiness.dart index 86313d49..49ff0a1d 100644 --- a/tool/testing/release_metadata_readiness.dart +++ b/tool/testing/release_metadata_readiness.dart @@ -38,11 +38,12 @@ typedef ReadinessFilePair = ({ReadinessFile base, ReadinessFile head}); String? validateReleaseMetadata(Map files) { if (!files.keys.toSet().containsAll({ 'pubspec.yaml', + releaseMetadataLock, ...releaseMetadataDocs, }) || files.keys.any((path) => !releaseMetadataPaths.contains(path))) { - return 'Metadata release requires core pubspec and both current changelogs; ' - 'only the fixed release metadata paths are allowed.'; + return 'Metadata release requires the complete fixed core pubspec, generated ' + 'chat lock and current release-document inventory.'; } for (final entry in files.entries) { if (entry.value.base.mode != '100644' || @@ -79,19 +80,8 @@ String? validateReleaseMetadata(Map files) { for (final path in releaseMetadataDocs) { final pair = files[path]; if (pair == null) continue; - // Markdown in Docusaurus may contain executable MDX. Preserve its complete - // syntax-bearing fragments and import/export lines rather than exempt it. - final baseActive = _activeFragments(pair.base.contents); - final headActive = _activeFragments(pair.head.contents); - if (baseActive == null || - headActive == null || - baseActive != headActive || - _frontMatter(pair.base.contents) != _frontMatter(pair.head.contents) || - _runtimeIdentities(pair.base.contents) != - _runtimeIdentities(pair.head.contents)) { - return 'Release documentation cannot change embedded MDX/HTML syntax: ' - '$path.'; - } + var baseProse = pair.base.contents; + var headProse = pair.head.contents; if (path.endsWith('README.md') || path.endsWith('installation.md')) { final dependencies = RegExp( r'^\s+llamadart:\s+\^([^\s#]+)', @@ -127,6 +117,25 @@ String? validateReleaseMetadata(Map files) { return 'Release changelogs must name the new patch and preserve all ' 'historical numbered sections: $path.'; } + // Historical bytes are already immutable. Do not reinterpret their old + // Markdown with a new parser; inspect only the mutable release prefix. + final historyLength = pair.base.contents.length - historical.start; + baseProse = pair.base.contents.substring(0, historical.start); + headProse = pair.head.contents.substring( + 0, + pair.head.contents.length - historyLength, + ); + } + // This exception is for Markdown prose/code examples, not active MDX. + // Deny active contexts altogether: fragment equality cannot establish JS + // equivalence (template literal attributes or script bodies can change). + if (_activeFragments(baseProse) != '' || + _activeFragments(headProse) != '' || + _frontMatter(pair.base.contents) != _frontMatter(pair.head.contents) || + _runtimeIdentities(pair.base.contents) != + _runtimeIdentities(pair.head.contents)) { + return 'Metadata release docs require inactive Markdown, unchanged ' + 'frontmatter and unchanged runtime identities: $path.'; } } return null; From a8bb6ff318d2ef028b81f549cd1aefe7929abf07 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Mon, 7 Sep 2026 16:18:33 -0400 Subject: [PATCH 3/3] test: guard release section transitions --- .../tooling/high_risk_readiness_test.dart | 20 +++++++++++++++++++ tool/testing/release_metadata_readiness.dart | 16 +++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/test/unit/tooling/high_risk_readiness_test.dart b/test/unit/tooling/high_risk_readiness_test.dart index 2e030c5b..489df22f 100644 --- a/test/unit/tooling/high_risk_readiness_test.dart +++ b/test/unit/tooling/high_risk_readiness_test.dart @@ -336,6 +336,26 @@ void mutateIdentityToOtherValidValues(Map evidence) { void main() { group('bounded metadata-only release evidence', () { + test( + 'new release prefix cannot duplicate or invent historical sections', + () async { + for (final heading in ['## 0.8.22', '## 9.0.0', '## Migration']) { + final files = metadataFiles(); + final old = files[headSha]!['CHANGELOG.md']!; + files[headSha]!['CHANGELOG.md'] = ( + mode: old.mode, + contents: old.contents.replaceFirst( + '* Update.', + '* Update.\n\n$heading\n\n* Duplicate.', + ), + ); + expectFailure( + await evaluateMetadata(files: files), + ReadinessFailureClassification.invalidReleaseMetadata, + ); + } + }, + ); test( 'existing active JSX template attributes and script bodies cannot use exception', () async { diff --git a/tool/testing/release_metadata_readiness.dart b/tool/testing/release_metadata_readiness.dart index 49ff0a1d..061913db 100644 --- a/tool/testing/release_metadata_readiness.dart +++ b/tool/testing/release_metadata_readiness.dart @@ -125,6 +125,22 @@ String? validateReleaseMetadata(Map files) { 0, pair.head.contents.length - historyLength, ); + final headings = RegExp(r'^## ([^\n]+)$', multiLine: true); + final before = headings + .allMatches(baseProse) + .map((m) => m.group(1)) + .toList(); + final after = headings + .allMatches(headProse) + .map((m) => m.group(1)) + .toList(); + if (before.length != 1 || + before.single != 'Unreleased' || + after.length != 1 || + after.single != newVersion) { + return 'Metadata release must replace only the Unreleased section ' + 'heading with the next core patch: $path.'; + } } // This exception is for Markdown prose/code examples, not active MDX. // Deny active contexts altogether: fragment equality cannot establish JS