From 7deaa32a8e77a661ff302aa93abc88de4b1b1d09 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 3 Sep 2026 00:14:26 +0530 Subject: [PATCH] test: ratchet accounting mutation evidence --- docs/knowledge/tooling-decisions.md | 6 +- .../stryker-accounting-oracle-2026-08-31.md | 46 ++++--- scripts/qualify-codex-accounting-oracle.mjs | 5 +- .../qualify-codex-accounting-oracle.test.mjs | 119 +++++++++++++++++- .../stryker-accounting-tap-trial.config.mjs | 15 +++ scripts/stryker-accounting.config.mjs | 6 +- 6 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 scripts/stryker-accounting-tap-trial.config.mjs diff --git a/docs/knowledge/tooling-decisions.md b/docs/knowledge/tooling-decisions.md index 7c46fa52..eac1c960 100644 --- a/docs/knowledge/tooling-decisions.md +++ b/docs/knowledge/tooling-decisions.md @@ -68,10 +68,10 @@ as a signed desktop sidecar remains **approved** work. | cargo-deny 0.20.2 | Wired | Offline Rust license, source, wildcard-requirement, and duplicate-version policy; native SARIF for actionable policy violations | | ast-grep 0.45.2 | Trialled, not wired | Structural locations were correct, but native `--format sarif` emits an invalid root format version; no missing-rule case justifies a converter yet | | Trivy 0.74.0 config scan | Trialled, not wired | Embedded checks found no supported first-party IaC surface; the unbounded scan targeted a dependency Dockerfile and the bounded scan misclassified a JSON fixture | -| OSV-Scanner 2.5.1 | Repository runner wired | `pnpm quality:vulnerabilities` produces offline SARIF plus a database-identity receipt; the 35-package baseline and remediation are tracked in issue #195 | -| StrykerJS 10.0.0 | Bounded local command wired | Accounting oracle: 218 mutants, 185 killed, 33 survived, 84.86% score; `pnpm quality:mutation:accounting`, tracked in issue #196 | +| OSV-Scanner 2.5.1 | Repository runner wired | `pnpm quality:vulnerabilities` produces fail-closed offline SARIF plus npm/crates.io/Go/SwiftURL database identities; the remediated baseline retains 20 result instances across 19 visible advisory IDs, tracked in issue #195 | +| StrykerJS 10.0.0 | Bounded local command wired | Accounting oracle: 208 mutants, 197 killed, 11 diagnostic/equivalent survivors, 94.71% score, and a 90% ratchet; the faster TAP runner was rejected because it left 66 mutants uncovered, tracked in issue #196 | | Schemathesis 4.25.2 | Rejected for current surface | CLI availability verified, but CodeVetter has no OpenAPI/Swagger contract or HTTP server to exercise | -| Apple `container` CLI | Approved for measured trial | Host qualifies (arm64, macOS 27), but the signed admin-installed system service is absent; tracked in issue #197 | +| Apple `container` CLI 1.3.1 | Trialled with containment defect | The signed/notarized service passed bounded no-network execution and cleanup, but accepted a controlled `..` sibling mount; any product adapter must canonicalize every mount under the selected root, tracked in issue #197 | | Lighthouse CI 0.15.1 | Trialled, rejected as a repo dependency | Three local landing-page runs passed the proposed category/Core Web Vitals gates, but the package introduced three high advisories including unpatched `extract-zip` traversal; raw Lighthouse JSON ingestion remains supported | | Size Limit 13.0.3 | Wired, additive | Caps the complete emitted desktop JS distribution after the existing Tauri-aware entry/Home and per-chunk budget gate; it does not replace those product-specific calculations | | `fast-xml-parser` 5.11.1 | Wired | Closed JUnit and Cobertura XML ingestion with DTD/entity rejection before parsing | diff --git a/evidence/verification/stryker-accounting-oracle-2026-08-31.md b/evidence/verification/stryker-accounting-oracle-2026-08-31.md index 0204cabe..19ddc602 100644 --- a/evidence/verification/stryker-accounting-oracle-2026-08-31.md +++ b/evidence/verification/stryker-accounting-oracle-2026-08-31.md @@ -4,27 +4,45 @@ StrykerJS was evaluated against `scripts/qualify-codex-accounting-oracle.mjs`, a deterministic verification boundary where a false pass would corrupt CodeVetter's accounting evidence. -| Measurement | Initial trial | Strengthened suite | -|---|---:|---:| -| StrykerJS | 10.0.0 | 10.0.0 | -| Mutants | 218 | 218 | -| Killed | 88 | 185 | -| Survived | 130 | 33 | -| Mutation score | 40.37% | 84.86% | -| Wall time | 14 seconds | 53 seconds | +| Measurement | Initial trial | Strengthened suite | Final ratchet | +|---|---:|---:|---:| +| StrykerJS | 10.0.0 | 10.0.0 | 10.0.0 | +| Mutants | 218 | 218 | 208 | +| Killed | 88 | 185 | 197 | +| Survived | 130 | 33 | 11 | +| Mutation score | 40.37% | 84.86% | 94.71% | +| Wall time | 14 seconds | 53 seconds | 125 seconds | The added tests cover invalid and zero numeric evidence, inverted cost bounds, provider selection, duplicate and missing daily buckets, CLI success/mismatch/ malformed-input exits, and the exact CodexBar subprocess arguments and `CODEX_HOME` handoff. -The 33 survivors are primarily error-message string changes and equivalent or -low-value implementation mutations. Remaining behavioral cases include exact -epsilon boundaries, deterministic multi-date ordering, and a forced CodexBar -non-zero exit. They remain visible in the ignored JSON report rather than being -excluded from mutation. +The final suite adds exact epsilon-boundary behavior, both out-of-range cost +directions, missing and invalid oracle days, deterministic multi-date ordering, +missing CLI input, the default executable path, and a forced CodexBar non-zero +exit. Five redundant/equivalent implementation expressions were removed rather +than excluded from mutation. The 11 remaining survivors are ten diagnostic-only +string/encoding changes and the equivalent `process.argv` index-zero mutation; +they remain visible in the ignored JSON report. The maintained local command is `pnpm quality:mutation:accounting`. It uses ephemeral, exactly-versioned StrykerJS and TypeScript packages, writes its report -under ignored `artifacts/tooling/stryker/`, and fails below 80%. It is deliberately +under ignored `artifacts/tooling/stryker/`, and fails below 90%. It is deliberately bounded to one high-value oracle rather than applied as a universal score. + +## TAP runner comparison + +The exactly matched `@stryker-mutator/tap-runner` 10.0.0 was trialled with the +same 218-mutant source and test file. Its mutation phase finished in 49 seconds, +but it killed only 136 mutants, left 15 survived, classified 66 as no-coverage, +and reported one error. Total score fell to 62.67% (90.07% over covered mutants) +because the oracle's subprocess CLI checks are outside TAP's per-test-file +coverage boundary. + +The faster runner therefore loses evidence precisely on the CLI behavior this +oracle exists to verify. The repository keeps the command runner, adds no TAP +plugin dependency, and retains +`scripts/stryker-accounting-tap-trial.config.mjs` only as a reproducible rejected +trial. The 90% break threshold is a score ratchet over the full 208-mutant +command-runner scope; no mutator exclusions were added to reach it. diff --git a/scripts/qualify-codex-accounting-oracle.mjs b/scripts/qualify-codex-accounting-oracle.mjs index e73bb87b..36a541cb 100644 --- a/scripts/qualify-codex-accounting-oracle.mjs +++ b/scripts/qualify-codex-accounting-oracle.mjs @@ -14,7 +14,7 @@ function integer(value, label) { } function cost(value, label) { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + if (!Number.isFinite(value) || value < 0) { throw new Error(`${label} must be a non-negative finite number`); } return value; @@ -65,7 +65,7 @@ export function normalizeCodexBar(raw) { export function normalizeCodeVetter(raw) { const totals = normalizeBucket(raw?.totals ?? {}, 'actual.totals'); const daily = new Map(); - for (const bucket of raw?.daily ?? []) { + for (const bucket of raw.daily ?? []) { if (typeof bucket.date !== 'string' || daily.has(bucket.date)) { throw new Error(`actual daily date is missing or duplicated: ${bucket.date}`); } @@ -130,7 +130,6 @@ function runOracle(codexHome, binary) { const result = spawnSync(binary, ['cost', '--provider', 'codex', '--json', '--refresh'], { encoding: 'utf8', env: { ...process.env, CODEX_HOME: codexHome }, - stdio: ['ignore', 'pipe', 'pipe'], }); if (result.status !== 0) { throw new Error(`CodexBar failed (${result.status}): ${result.stderr.trim()}`); diff --git a/scripts/qualify-codex-accounting-oracle.test.mjs b/scripts/qualify-codex-accounting-oracle.test.mjs index c0ce03a8..6c673147 100644 --- a/scripts/qualify-codex-accounting-oracle.test.mjs +++ b/scripts/qualify-codex-accounting-oracle.test.mjs @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, copyFileSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test, { after, before } from 'node:test'; @@ -56,6 +56,7 @@ const actual = { let fixtureDirectory; let fakeCodexBarPath; +let failingCodexBarPath; const scriptPath = fileURLToPath(new URL('./qualify-codex-accounting-oracle.mjs', import.meta.url)); before(() => { @@ -77,6 +78,18 @@ process.stdout.write(${JSON.stringify(JSON.stringify(oracle))}); ` ); chmodSync(fakeCodexBarPath, 0o755); + const defaultCodexBarPath = join(fixtureDirectory, 'codexbar'); + copyFileSync(fakeCodexBarPath, defaultCodexBarPath); + chmodSync(defaultCodexBarPath, 0o755); + failingCodexBarPath = join(fixtureDirectory, 'failing-codexbar.mjs'); + writeFileSync( + failingCodexBarPath, + `#!/usr/bin/env node +process.stderr.write('controlled oracle failure\\n'); +process.exit(23); +` + ); + chmodSync(failingCodexBarPath, 0o755); }); after(() => { @@ -115,6 +128,40 @@ test('accepts an oracle cost inside bounded service-tier uncertainty', () => { assert.deepEqual(compareAccounting(normalizeCodexBar(oracle), normalizeCodeVetter(ranged)), []); }); +test('accepts exact epsilon boundaries and rejects either outside direction', () => { + const epsilon = 1e-12; + const boundary = structuredClone(actual); + delete boundary.totals.api_equivalent_cost_usd; + boundary.totals.api_equivalent_cost_min_usd = oracle.totals.totalCost + epsilon; + boundary.totals.api_equivalent_cost_max_usd = oracle.totals.totalCost + epsilon; + assert.deepEqual(compareAccounting(normalizeCodexBar(oracle), normalizeCodeVetter(boundary)), []); + + const lowerBoundary = structuredClone(actual); + delete lowerBoundary.totals.api_equivalent_cost_usd; + lowerBoundary.totals.api_equivalent_cost_min_usd = oracle.totals.totalCost - epsilon; + lowerBoundary.totals.api_equivalent_cost_max_usd = oracle.totals.totalCost - epsilon; + assert.deepEqual( + compareAccounting(normalizeCodexBar(oracle), normalizeCodeVetter(lowerBoundary)), + [] + ); + + const above = structuredClone(boundary); + above.totals.api_equivalent_cost_min_usd = oracle.totals.totalCost + epsilon * 2; + above.totals.api_equivalent_cost_max_usd = oracle.totals.totalCost + epsilon * 3; + assert.equal( + compareAccounting(normalizeCodexBar(oracle), normalizeCodeVetter(above))[0].field, + 'api_equivalent_cost_usd' + ); + + const below = structuredClone(boundary); + below.totals.api_equivalent_cost_min_usd = oracle.totals.totalCost - epsilon * 3; + below.totals.api_equivalent_cost_max_usd = oracle.totals.totalCost - epsilon * 2; + assert.equal( + compareAccounting(normalizeCodexBar(oracle), normalizeCodeVetter(below))[0].field, + 'api_equivalent_cost_usd' + ); +}); + test('rejects an oracle cost outside CodeVetter bounds', () => { const changed = structuredClone(actual); changed.totals.api_equivalent_cost_usd = 0.2; @@ -192,9 +239,32 @@ test('requires the local Codex provider and valid unique daily dates', () => { const duplicateOracleDate = structuredClone(oracle); duplicateOracleDate.daily.push(structuredClone(oracle.daily[0])); assert.throws(() => normalizeCodexBar(duplicateOracleDate), /oracle daily.*duplicated/); + const invalidOracleDate = structuredClone(oracle); + invalidOracleDate.daily[0].date = 20260716; + assert.throws(() => normalizeCodexBar(invalidOracleDate), /oracle daily.*duplicated/); + assert.deepEqual([...normalizeCodexBar({ ...oracle, daily: undefined }).daily], []); assert.throws(() => normalizeCodeVetter(undefined), /actual\.totals\.input_tokens/); }); +test('sorts multi-date comparison output deterministically', () => { + const orderedOracle = structuredClone(oracle); + orderedOracle.daily = [ + { ...structuredClone(oracle.daily[0]), date: '2026-07-18' }, + { ...structuredClone(oracle.daily[0]), date: '2026-07-16' }, + ]; + const orderedActual = structuredClone(actual); + orderedActual.daily = [ + { ...structuredClone(actual.daily[0]), date: '2026-07-17' }, + { ...structuredClone(actual.daily[0]), date: '2026-07-16' }, + ]; + assert.deepEqual( + compareAccounting(normalizeCodexBar(orderedOracle), normalizeCodeVetter(orderedActual)).map( + ({ scope }) => scope + ), + ['daily.2026-07-17', 'daily.2026-07-18'] + ); +}); + test('reports missing daily buckets in either input', () => { const missingActual = structuredClone(actual); missingActual.daily = []; @@ -265,6 +335,26 @@ test('CLI distinguishes qualified, mismatched, and malformed evidence', () => { ); assert.equal(malformed.status, 2); assert.match(malformed.stderr, /JSON/); + + const missingActualPath = spawnSync(process.execPath, [scriptPath, '--oracle-json', oraclePath], { + encoding: 'utf8', + }); + assert.equal(missingActualPath.status, 2); + assert.match(missingActualPath.stderr, /--codevetter-json path is required/); + + const multiDateOracle = structuredClone(oracle); + multiDateOracle.daily.unshift({ ...structuredClone(oracle.daily[0]), date: '2026-07-18' }); + const multiDateActual = structuredClone(actual); + multiDateActual.daily.unshift({ ...structuredClone(actual.daily[0]), date: '2026-07-18' }); + writeFileSync(oraclePath, JSON.stringify(multiDateOracle)); + writeFileSync(actualPath, JSON.stringify(multiDateActual)); + const ordered = spawnSync( + process.execPath, + [scriptPath, '--oracle-json', oraclePath, '--codevetter-json', actualPath], + { encoding: 'utf8' } + ); + assert.equal(ordered.status, 0, ordered.stderr); + assert.deepEqual(JSON.parse(ordered.stdout).compared_daily_buckets, ['2026-07-16', '2026-07-18']); }); test('CLI invokes CodexBar with its exact local evidence contract', () => { @@ -288,6 +378,17 @@ test('CLI invokes CodexBar with its exact local evidence contract', () => { assert.equal(qualified.status, 0, qualified.stderr); assert.equal(JSON.parse(qualified.stdout).qualified, true); + const defaultBinary = spawnSync( + process.execPath, + [scriptPath, '--codex-home', codexHome, '--codevetter-json', actualPath], + { + encoding: 'utf8', + env: { ...process.env, PATH: `${fixtureDirectory}:${process.env.PATH ?? ''}` }, + } + ); + assert.equal(defaultBinary.status, 0, defaultBinary.stderr); + assert.equal(JSON.parse(defaultBinary.stdout).qualified, true); + const missingHome = spawnSync( process.execPath, [scriptPath, '--codevetter-json', actualPath, '--codexbar', fakeCodexBarPath], @@ -295,4 +396,20 @@ test('CLI invokes CodexBar with its exact local evidence contract', () => { ); assert.equal(missingHome.status, 2); assert.match(missingHome.stderr, /--codex-home is required/); + + const failedOracle = spawnSync( + process.execPath, + [ + scriptPath, + '--codex-home', + codexHome, + '--codexbar', + failingCodexBarPath, + '--codevetter-json', + actualPath, + ], + { encoding: 'utf8' } + ); + assert.equal(failedOracle.status, 2); + assert.equal(failedOracle.stderr, 'CodexBar failed (23): controlled oracle failure\n'); }); diff --git a/scripts/stryker-accounting-tap-trial.config.mjs b/scripts/stryker-accounting-tap-trial.config.mjs new file mode 100644 index 00000000..d2a7986f --- /dev/null +++ b/scripts/stryker-accounting-tap-trial.config.mjs @@ -0,0 +1,15 @@ +import commandRunnerConfig from './stryker-accounting.config.mjs'; + +export default { + ...commandRunnerConfig, + plugins: ['@stryker-mutator/tap-runner'], + testRunner: 'tap', + tap: { + testFiles: ['scripts/qualify-codex-accounting-oracle.test.mjs'], + forceBail: true, + }, + coverageAnalysis: 'perTest', + jsonReporter: { + fileName: 'artifacts/tooling/stryker/accounting-tap-mutation-report.json', + }, +}; diff --git a/scripts/stryker-accounting.config.mjs b/scripts/stryker-accounting.config.mjs index 9d7af3f7..11d66367 100644 --- a/scripts/stryker-accounting.config.mjs +++ b/scripts/stryker-accounting.config.mjs @@ -26,8 +26,8 @@ export default { fileName: 'artifacts/tooling/stryker/accounting-mutation-report.json', }, thresholds: { - high: 80, - low: 60, - break: 80, + high: 90, + low: 80, + break: 90, }, };