From fa94292eb8b8d7c0a7ec3cd577a1e734edfbe956 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 11:58:37 +0000 Subject: [PATCH 01/16] feat(addie): add fixed-trace experiment plan gate --- .../addie/eval/fixed-trace-experiment-plan.ts | 595 ++++++++++++++++++ .../src/addie/eval/fixed-trace-partition.ts | 50 ++ server/src/addie/model-cost-pricing.ts | 29 + .../openai-responses-provider.ts | 20 +- .../tests/manual/fixed-trace-provider-eval.ts | 37 +- .../addie/fixed-trace-experiment-plan.test.ts | 217 +++++++ .../model-provider-openai-google.test.ts | 9 + 7 files changed, 953 insertions(+), 4 deletions(-) create mode 100644 server/src/addie/eval/fixed-trace-experiment-plan.ts create mode 100644 server/src/addie/eval/fixed-trace-partition.ts create mode 100644 server/tests/unit/addie/fixed-trace-experiment-plan.test.ts diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts new file mode 100644 index 0000000000..87c2be3d80 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-experiment-plan.ts @@ -0,0 +1,595 @@ +import { createHash } from 'node:crypto'; +import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; +import { + GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS, + OPENAI_GPT_5_6_PRICING_VERSION, +} from '../model-cost-pricing.js'; +import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; +import { + FIXED_TRACE_PARTITION_MANIFEST, + FIXED_TRACE_PARTITION_MANIFEST_SHA256, + FIXED_TRACE_PARTITION_MANIFEST_VERSION, + assertFixedTracePartitionManifest, +} from './fixed-trace-partition.js'; + +/** A versioned, network-free admission contract for fixed-trace experiments. */ +export const FIXED_TRACE_EXPERIMENT_PLAN_VERSION = 'addie-fixed-trace-experiment-plan-v1' as const; +export const FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION = + 'addie-fixed-trace-holdout-finalization-v1' as const; +export const FIXED_TRACE_RAW_LEDGER_VERSION = 'addie-fixed-trace-raw-ledger-v1' as const; +export const FIXED_TRACE_HOLDOUT_LIMITATION = + 'execution_locked_repository_visible_not_secret_holdout' as const; + +export type FixedTraceExperimentArchitecture = + | 'two_stage_llm_router' + | 'direct_generation' + | 'hybrid_generation' + | 'oracle_route_diagnostic'; +export type FixedTraceScreeningStage = + | 'router_only_screen' + | 'oracle_route_generator_diagnostic' + | 'deployable_finalist'; + +export interface FixedTraceImmutablePricingProfile { + provider: ModelProviderId; + model: string; + version: string; + validBefore: string; + inputUsdPerMillionTokens: number; + outputUsdPerMillionTokens: number; + source: string; +} + +/** + * Profiles are deliberately a closed list. A missing provider/model/version is + * unavailable, rather than inheriting a sibling model's price. + */ +export const FIXED_TRACE_IMMUTABLE_PRICING = Object.freeze([ + Object.freeze({ + provider: 'anthropic', model: 'claude-haiku-4-5', version: CLAUDE_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, + source: 'Repository Anthropic standard pricing table, refreshed August 2026.', + }), + Object.freeze({ + provider: 'anthropic', model: 'claude-sonnet-5', version: CLAUDE_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 3, outputUsdPerMillionTokens: 15, + source: 'Repository Anthropic standard pricing table, refreshed August 2026.', + }), + Object.freeze({ + provider: 'openai', model: 'gpt-5.6-luna', version: OPENAI_GPT_5_6_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-luna'].inputUsd, + outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-luna'].outputUsd, + source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', + }), + Object.freeze({ + provider: 'openai', model: 'gpt-5.6-terra', version: OPENAI_GPT_5_6_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-terra'].inputUsd, + outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-terra'].outputUsd, + source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', + }), + Object.freeze({ + provider: 'openai', model: 'gpt-5.6-sol', version: OPENAI_GPT_5_6_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-sol'].inputUsd, + outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-sol'].outputUsd, + source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', + }), + Object.freeze({ + provider: 'google', model: 'gemini-3.7-flash', version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + validBefore: '2027-01-01T00:00:00.000Z', inputUsdPerMillionTokens: 0.75, outputUsdPerMillionTokens: 3.75, + source: 'Repository Google Gemini 3.7 Flash pricing pin through 2026-12-31.', + }), +] satisfies readonly FixedTraceImmutablePricingProfile[]); + +export interface FixedTraceRequestBounds { + /** One exact UTF-8 request byte count for every possible request in the loop. */ + inputBytesByTrace: Readonly>; +} + +export interface FixedTracePlannedStage { + provider: ModelProviderId; + model: string; + reasoningEffort: ModelReasoningEffort; + pricingVersion: string; + maxOutputTokens: number; + timeoutMs: number; + maxIterations: number; + samplingMode: 'temperature_zero' | 'provider_no_sampling_control'; + temperature: 0 | null; + requestBounds: FixedTraceRequestBounds; +} + +export interface FixedTracePlannedJudge extends FixedTracePlannedStage { + blinded: true; +} + +export interface FixedTraceExperimentArm { + id: string; + architecture: FixedTraceExperimentArchitecture; + screeningStage: FixedTraceScreeningStage; + repetitionIndex: number; + router?: FixedTracePlannedStage; + generation?: FixedTracePlannedStage; + judges?: readonly FixedTracePlannedJudge[]; +} + +export interface FixedTraceExperimentPlan { + version: typeof FIXED_TRACE_EXPERIMENT_PLAN_VERSION; + id: string; + /** Resolved outside the candidate-controlled plan before it is admissible. */ + trustedManifestId: string; + sourceId: string; + sourceRevision: string; + pricingAsOf: string; + sourceBundleSha256: string; + traceSuiteSha256: string; + promptConfigVersion: string; + toolSchemaSha256: string; + partition: { + manifestVersion: typeof FIXED_TRACE_PARTITION_MANIFEST_VERSION; + manifestSha256: typeof FIXED_TRACE_PARTITION_MANIFEST_SHA256; + selected: 'development' | 'holdout'; + /** Holdout is legal only for a separately versioned, explicit finalization. */ + finalizationGate?: { version: typeof FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION; recordId: string }; + }; + ordering: { seed: string }; + budgets: { candidateCeilingUsd: number; judgeCeilingUsd: number }; + arms: readonly FixedTraceExperimentArm[]; +} + +/** + * The resolver is deliberately external to the plan file. A plan cannot make + * itself trusted by repeating its own hashes. The future dispatcher must use + * an attested/controlled resolver, never deserialize this alongside a plan. + */ +export interface FixedTraceTrustedManifest { + id: string; + sourceId: string; + sourceRevision: string; + sourceBundleSha256: string; + traceSuiteSha256: string; + promptConfigVersion: string; + toolSchemaSha256: string; + partitionManifestSha256: string; + rawLedgerVersion: typeof FIXED_TRACE_RAW_LEDGER_VERSION; +} + +export type FixedTraceTrustedManifestResolver = (id: string) => FixedTraceTrustedManifest | null; + +/** + * Finalization state belongs to a controlled store, not the candidate plan. + * `consume` is intentionally separate from inspection: dry runs never spend + * the one-time holdout authorization. + */ +export interface FixedTraceHoldoutFinalizationRecord { + id: string; + version: typeof FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION; + trustedManifestId: string; + frozenCandidatePlanFingerprint: string; + consumed: boolean; + tracePackVisibility: 'repository_visible' | 'externally_sealed'; +} +export type FixedTraceHoldoutFinalizationResolver = (id: string) => FixedTraceHoldoutFinalizationRecord | null; +export type FixedTraceHoldoutFinalizationConsumer = (id: string, frozenCandidatePlanFingerprint: string) => boolean; + +export interface FixedTraceRawLedgerEntry { + sequence: number; + armId: string; + repetitionIndex: number; + traceId: string; + stage: 'router' | 'generation' | 'judge'; + dispatched: boolean; + requestedProvider: ModelProviderId | null; + requestedModel: string | null; + returnedProvider: ModelProviderId | null; + returnedModel: string | null; + promptSha256: string; + providerRequestSha256: string | null; + responseSha256: string | null; + /** Content-addressed immutable raw artifacts; their bytes stay outside summaries. */ + rawRequestArtifact: { sha256: string; byteLength: number; storageKey: string } | null; + rawResponseArtifact: { sha256: string; byteLength: number; storageKey: string } | null; + exactToolNames: readonly string[]; + caseControlSha256: string; + executionEnvelopeSha256: string; + directAdmissionSha256: string; + maxOutputTokens: number | null; + timeoutMs: number | null; + maxIterations: number | null; + reasoningEffort: ModelReasoningEffort; + samplingMode: 'temperature_zero' | 'provider_no_sampling_control' | null; +} + +export interface FixedTraceRawAuditableLedger { + version: typeof FIXED_TRACE_RAW_LEDGER_VERSION; + trustedManifestSha256: string; + planFingerprint: string; + entries: readonly FixedTraceRawLedgerEntry[]; +} +export type FixedTraceRawArtifactResolver = (storageKey: string) => { sha256: string; byteLength: number } | null; + +export interface FixedTraceStageReservation { + armId: string; + repetitionIndex: number; + stage: 'router' | 'generation' | 'judge'; + provider: ModelProviderId; + model: string; + requests: number; + inputBytes: number; + outputTokens: number; + ceilingUsd: number; +} + +export interface FixedTraceDryRunEstimate { + planFingerprint: string; + executionOrder: readonly string[]; + candidate: { ceilingUsd: number; expectedSpendUsd: null; reservations: readonly FixedTraceStageReservation[] }; + judges: { ceilingUsd: number; expectedSpendUsd: null; reservations: readonly FixedTraceStageReservation[] }; + totalCeilingUsd: number; + /** No traffic or provider calls occur; expected spend needs observed usage. */ + expectedSpendUsd: null; +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error('Cannot fingerprint a non-finite experiment-plan value'); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; + } + throw new Error('Cannot fingerprint a non-JSON experiment-plan value'); +} + +function sha256(value: unknown): string { + return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); +} + +function requireHash(value: string, label: string): void { + if (!/^[a-f0-9]{64}$/.test(value)) throw new Error(`${label} must be a SHA-256 hex digest`); +} + +function requirePositiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`); +} + +function selectedTraceIds(plan: FixedTraceExperimentPlan): readonly string[] { + return FIXED_TRACE_PARTITION_MANIFEST[plan.partition.selected]; +} + +function pricingFor(stage: FixedTracePlannedStage, pricingAsOf: string): FixedTraceImmutablePricingProfile { + const asOf = new Date(pricingAsOf); + if (Number.isNaN(asOf.getTime())) throw new Error('pricingAsOf must be an ISO timestamp'); + const pricing = FIXED_TRACE_IMMUTABLE_PRICING.find((candidate) => + candidate.provider === stage.provider && candidate.model === stage.model && candidate.version === stage.pricingVersion, + ); + if (!pricing) throw new Error(`Unavailable immutable pricing for ${stage.provider}/${stage.model}`); + if (asOf >= new Date(pricing.validBefore)) { + throw new Error(`Stale immutable pricing for ${stage.provider}/${stage.model}`); + } + return pricing; +} + +function validateStage( + stage: FixedTracePlannedStage, + label: string, + traceIds: readonly string[], + pricingAsOf: string, +): FixedTraceImmutablePricingProfile { + if (!stage.model.trim()) throw new Error(`${label}.model is required`); + requirePositiveInteger(stage.maxOutputTokens, `${label}.maxOutputTokens`); + requirePositiveInteger(stage.timeoutMs, `${label}.timeoutMs`); + requirePositiveInteger(stage.maxIterations, `${label}.maxIterations`); + if ( + (stage.samplingMode === 'temperature_zero' && stage.temperature !== 0) + || (stage.samplingMode === 'provider_no_sampling_control' && stage.temperature !== null) + ) throw new Error(`${label} sampling controls are inconsistent`); + const pricing = pricingFor(stage, pricingAsOf); + const bounds = stage.requestBounds?.inputBytesByTrace; + if (!bounds || typeof bounds !== 'object') throw new Error(`${label}.requestBounds are required`); + for (const traceId of traceIds) { + const values = bounds[traceId]; + if (!Array.isArray(values) || values.length !== stage.maxIterations) { + throw new Error(`${label}.requestBounds must contain ${stage.maxIterations} exact bounds for ${traceId}`); + } + for (const bytes of values) requirePositiveInteger(bytes, `${label}.requestBounds.${traceId}`); + } + if (Object.keys(bounds).some((traceId) => !traceIds.includes(traceId))) { + throw new Error(`${label}.requestBounds contains a trace outside the selected partition`); + } + return pricing; +} + +function validateArm(plan: FixedTraceExperimentPlan, arm: FixedTraceExperimentArm): void { + if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid experiment arm ID: ${arm.id}`); + requirePositiveInteger(arm.repetitionIndex, `${arm.id}.repetitionIndex`); + const traces = selectedTraceIds(plan); + const stages = arm.screeningStage; + if (stages === 'router_only_screen') { + if (arm.architecture !== 'two_stage_llm_router' || !arm.router || arm.generation || (arm.judges?.length ?? 0) !== 0) { + throw new Error(`${arm.id} is not a router-only screening contract`); + } + validateStage(arm.router, `${arm.id}.router`, traces, plan.pricingAsOf); + return; + } + if (stages === 'oracle_route_generator_diagnostic') { + if (arm.architecture !== 'oracle_route_diagnostic' || arm.router || !arm.generation || (arm.judges?.length ?? 0) !== 0) { + throw new Error(`${arm.id} is not an oracle-route diagnostic contract`); + } + validateStage(arm.generation, `${arm.id}.generation`, traces, plan.pricingAsOf); + return; + } + if (arm.architecture !== 'two_stage_llm_router' || !arm.router || !arm.generation) { + throw new Error(`${arm.id} is inadmissible: direct and hybrid execution contracts are not available`); + } + validateStage(arm.router, `${arm.id}.router`, traces, plan.pricingAsOf); + validateStage(arm.generation, `${arm.id}.generation`, traces, plan.pricingAsOf); + if (!arm.judges || arm.judges.length < 2) throw new Error(`${arm.id} requires at least two blinded independent judges`); + const candidateProviders = new Set([arm.router.provider, arm.generation.provider]); + const judgeProviders = new Set(); + for (const [index, judge] of arm.judges.entries()) { + if (judge.blinded !== true) throw new Error(`${arm.id}.judges.${index} must be blinded`); + if (candidateProviders.has(judge.provider)) throw new Error(`${arm.id}.judges.${index} is not provider-independent`); + judgeProviders.add(judge.provider); + validateStage(judge, `${arm.id}.judges.${index}`, traces, plan.pricingAsOf); + } + if (judgeProviders.size < 2) throw new Error(`${arm.id} requires two provider-independent judges`); +} + +/** Validates without loading trace fixtures, prompts, credentials, or providers. */ +function resolveTrustedManifest( + plan: FixedTraceExperimentPlan, + resolver: FixedTraceTrustedManifestResolver, +): FixedTraceTrustedManifest { + const manifest = resolver(plan.trustedManifestId); + if (!manifest) throw new Error(`Trusted fixed-trace manifest is unavailable: ${plan.trustedManifestId}`); + requireHash(manifest.sourceBundleSha256, 'trusted manifest sourceBundleSha256'); + requireHash(manifest.traceSuiteSha256, 'trusted manifest traceSuiteSha256'); + requireHash(manifest.promptConfigVersion, 'trusted manifest promptConfigVersion'); + requireHash(manifest.toolSchemaSha256, 'trusted manifest toolSchemaSha256'); + if (manifest.rawLedgerVersion !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Trusted manifest requires an unsupported raw ledger'); + if ( + manifest.id !== plan.trustedManifestId + || manifest.sourceId !== plan.sourceId + || manifest.sourceRevision !== plan.sourceRevision + || manifest.sourceBundleSha256 !== plan.sourceBundleSha256 + || manifest.traceSuiteSha256 !== plan.traceSuiteSha256 + || manifest.promptConfigVersion !== plan.promptConfigVersion + || manifest.toolSchemaSha256 !== plan.toolSchemaSha256 + || manifest.partitionManifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256 + ) throw new Error('Experiment plan does not match its trusted manifest'); + return manifest; +} + +/** Omits only execution partition/finalization state so an approved candidate cannot drift at unlock. */ +export function fixedTraceCandidatePlanFingerprint(plan: FixedTraceExperimentPlan): string { + const { partition, ...candidatePlan } = plan; + return sha256({ + ...candidatePlan, + partition: { + manifestVersion: partition.manifestVersion, + manifestSha256: partition.manifestSha256, + }, + }); +} + +function assertHoldoutFinalization( + plan: FixedTraceExperimentPlan, + resolver: FixedTraceHoldoutFinalizationResolver | undefined, +): void { + if (plan.partition.selected !== 'holdout') return; + const gate = plan.partition.finalizationGate; + if (!gate || gate.version !== FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION) { + throw new Error('Holdout is locked; an explicit versioned finalization gate is required'); + } + if (!resolver) throw new Error('Holdout is locked; an externally resolved finalization record is required'); + const record = resolver(gate.recordId); + if (!record) throw new Error(`Holdout finalization record is unavailable: ${gate.recordId}`); + if ( + record.version !== FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION + || record.trustedManifestId !== plan.trustedManifestId + || record.frozenCandidatePlanFingerprint !== fixedTraceCandidatePlanFingerprint(plan) + ) throw new Error('Holdout finalization record does not match the frozen candidate plan'); + if (record.consumed) throw new Error('Holdout finalization record has already been consumed'); +} + +export function assertFixedTraceExperimentPlan( + plan: FixedTraceExperimentPlan, + resolver: FixedTraceTrustedManifestResolver, + holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, +): void { + assertFixedTracePartitionManifest(); + if (plan.version !== FIXED_TRACE_EXPERIMENT_PLAN_VERSION) throw new Error('Unsupported fixed-trace experiment plan version'); + if (!plan.id.trim()) throw new Error('Experiment plan ID is required'); + if (!plan.trustedManifestId.trim() || !plan.sourceId.trim() || !plan.sourceRevision.trim()) throw new Error('Experiment plan requires a trusted source identity'); + requireHash(plan.sourceBundleSha256, 'sourceBundleSha256'); + requireHash(plan.traceSuiteSha256, 'traceSuiteSha256'); + requireHash(plan.promptConfigVersion, 'promptConfigVersion'); + requireHash(plan.toolSchemaSha256, 'toolSchemaSha256'); + if (plan.partition.manifestVersion !== FIXED_TRACE_PARTITION_MANIFEST_VERSION || plan.partition.manifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { + throw new Error('Experiment plan uses an uncommitted fixed-trace partition manifest'); + } + if (plan.partition.selected === 'holdout') { + assertHoldoutFinalization(plan, holdoutFinalizationResolver); + } else if (plan.partition.finalizationGate) { + throw new Error('Development execution must not carry a holdout finalization gate'); + } + if (!plan.ordering.seed.trim()) throw new Error('Experiment ordering seed is required'); + if (!Number.isFinite(plan.budgets.candidateCeilingUsd) || plan.budgets.candidateCeilingUsd <= 0) throw new Error('candidateCeilingUsd must be positive'); + if (!Number.isFinite(plan.budgets.judgeCeilingUsd) || plan.budgets.judgeCeilingUsd <= 0) throw new Error('judgeCeilingUsd must be positive'); + if (!Array.isArray(plan.arms) || plan.arms.length === 0) throw new Error('Experiment plan requires at least one arm'); + const ids = new Set(); + for (const arm of plan.arms) { + if (ids.has(arm.id)) throw new Error(`Duplicate experiment arm ID: ${arm.id}`); + ids.add(arm.id); + validateArm(plan, arm); + } + resolveTrustedManifest(plan, resolver); +} + +export function fixedTraceExperimentPlanFingerprint(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): string { + assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + return sha256(plan); +} + +/** Deterministic permutation based on a recorded seed, never provider input order. */ +export function fixedTraceExperimentExecutionOrder(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): readonly string[] { + assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + return Object.freeze([...plan.arms] + .sort((left, right) => sha256({ seed: plan.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) + .localeCompare(sha256({ seed: plan.ordering.seed, arm: right.id, repetition: right.repetitionIndex })) || left.id.localeCompare(right.id)) + .map((arm) => arm.id)); +} + +function reservation( + arm: FixedTraceExperimentArm, + stageName: FixedTraceStageReservation['stage'], + stage: FixedTracePlannedStage, + traceIds: readonly string[], + pricingAsOf: string, +): FixedTraceStageReservation { + const pricing = pricingFor(stage, pricingAsOf); + const inputBytes = traceIds.reduce((total, traceId) => total + stage.requestBounds.inputBytesByTrace[traceId].reduce((sum, bytes) => sum + bytes, 0), 0); + const requests = traceIds.length * stage.maxIterations; + const outputTokens = requests * stage.maxOutputTokens; + const ceilingUsd = ( + inputBytes * pricing.inputUsdPerMillionTokens + outputTokens * pricing.outputUsdPerMillionTokens + ) / 1_000_000; + return Object.freeze({ armId: arm.id, repetitionIndex: arm.repetitionIndex, stage: stageName, provider: stage.provider, model: stage.model, requests, inputBytes, outputTokens, ceilingUsd }); +} + +/** + * Pure pre-dispatch ceiling. It reports no expected spend because neither + * provider tokenization nor observed tool-loop length may be assumed. + */ +export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): FixedTraceDryRunEstimate { + assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + const candidate: FixedTraceStageReservation[] = []; + const judges: FixedTraceStageReservation[] = []; + const traceIds = selectedTraceIds(plan); + for (const arm of plan.arms) { + if (arm.router) candidate.push(reservation(arm, 'router', arm.router, traceIds, plan.pricingAsOf)); + if (arm.generation) candidate.push(reservation(arm, 'generation', arm.generation, traceIds, plan.pricingAsOf)); + for (const judge of arm.judges ?? []) judges.push(reservation(arm, 'judge', judge, traceIds, plan.pricingAsOf)); + } + const candidateCeilingUsd = candidate.reduce((total, item) => total + item.ceilingUsd, 0); + const judgeCeilingUsd = judges.reduce((total, item) => total + item.ceilingUsd, 0); + if (candidateCeilingUsd > plan.budgets.candidateCeilingUsd) throw new Error('Candidate worst-case reservation exceeds its separate budget'); + if (judgeCeilingUsd > plan.budgets.judgeCeilingUsd) throw new Error('Judge worst-case reservation exceeds its separate budget'); + return Object.freeze({ + planFingerprint: fixedTraceExperimentPlanFingerprint(plan, resolver, holdoutFinalizationResolver), + executionOrder: fixedTraceExperimentExecutionOrder(plan, resolver, holdoutFinalizationResolver), + candidate: Object.freeze({ ceilingUsd: candidateCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(candidate) }), + judges: Object.freeze({ ceilingUsd: judgeCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(judges) }), + totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd, + expectedSpendUsd: null, + }); +} + +/** ID-only audit output; callers must not load holdout expectations into prompts. */ +export function fixedTraceExperimentPartitionAudit(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): { selected: 'development' | 'holdout'; traceIds: readonly string[]; manifestSha256: string; blindingLimitation: typeof FIXED_TRACE_HOLDOUT_LIMITATION } { + assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + return Object.freeze({ selected: plan.partition.selected, traceIds: Object.freeze([...selectedTraceIds(plan)]), manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, blindingLimitation: FIXED_TRACE_HOLDOUT_LIMITATION }); +} + +/** Development selection artifacts cannot contain holdout metrics or IDs. */ +export function fixedTraceDevelopmentSelectionArtifact(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): { planFingerprint: string; developmentTraceIds: readonly string[]; holdoutMetricsIncluded: false; blindingLimitation: typeof FIXED_TRACE_HOLDOUT_LIMITATION } { + assertFixedTraceExperimentPlan(plan, resolver); + if (plan.partition.selected !== 'development') throw new Error('Holdout results cannot be emitted as a development selection artifact'); + return Object.freeze({ planFingerprint: fixedTraceExperimentPlanFingerprint(plan, resolver), developmentTraceIds: Object.freeze([...FIXED_TRACE_PARTITION_MANIFEST.development]), holdoutMetricsIncluded: false, blindingLimitation: FIXED_TRACE_HOLDOUT_LIMITATION }); +} + +/** Must be called by a future dispatcher immediately before the first holdout dispatch. */ +export function consumeFixedTraceHoldoutFinalization( + plan: FixedTraceExperimentPlan, + resolver: FixedTraceTrustedManifestResolver, + finalizationResolver: FixedTraceHoldoutFinalizationResolver, + consumer: FixedTraceHoldoutFinalizationConsumer, +): void { + assertFixedTraceExperimentPlan(plan, resolver, finalizationResolver); + if (plan.partition.selected !== 'holdout') throw new Error('Only a holdout plan can consume finalization'); + const recordId = plan.partition.finalizationGate!.recordId; + if (!consumer(recordId, fixedTraceCandidatePlanFingerprint(plan))) throw new Error('Holdout finalization record could not be consumed'); +} + +/** + * Validates raw-auditable execution provenance before anything can be used in + * comparison or rollout. This does not score or promote a candidate: the + * repaired foundation owns evidence verification and must consume this ledger. + */ +export function assertFixedTraceRawAuditableLedger( + plan: FixedTraceExperimentPlan, + resolver: FixedTraceTrustedManifestResolver, + ledger: FixedTraceRawAuditableLedger, + artifactResolver: FixedTraceRawArtifactResolver, + holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, +): void { + const manifest = resolveTrustedManifest(plan, resolver); + assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + if (ledger.version !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Unsupported raw fixed-trace ledger version'); + if (ledger.trustedManifestSha256 !== sha256(manifest)) throw new Error('Raw ledger trusted manifest mismatch'); + if (ledger.planFingerprint !== fixedTraceExperimentPlanFingerprint(plan, resolver, holdoutFinalizationResolver)) throw new Error('Raw ledger plan fingerprint mismatch'); + const knownArms = new Map(plan.arms.map((arm) => [arm.id, arm])); + const knownTraces = new Set(selectedTraceIds(plan)); + const expectedEntries = new Set(); + for (const arm of plan.arms) for (const traceId of selectedTraceIds(plan)) { + if (arm.router) expectedEntries.add(`${arm.id}\0${arm.repetitionIndex}\0${traceId}\0router`); + if (arm.generation) expectedEntries.add(`${arm.id}\0${arm.repetitionIndex}\0${traceId}\0generation`); + for (const judge of arm.judges ?? []) expectedEntries.add(`${arm.id}\0${arm.repetitionIndex}\0${traceId}\0judge\0${judge.provider}\0${judge.model}`); + } + const entries = new Set(); + for (const entry of ledger.entries) { + requirePositiveInteger(entry.sequence, 'raw ledger sequence'); + const arm = knownArms.get(entry.armId); + if (!arm || arm.repetitionIndex !== entry.repetitionIndex || !knownTraces.has(entry.traceId)) throw new Error('Raw ledger entry is outside its trusted plan'); + const configuredStage = entry.stage === 'router' ? arm.router + : entry.stage === 'generation' ? arm.generation + : (arm.judges ?? []).find((judge) => judge.provider === entry.requestedProvider && judge.model === entry.requestedModel); + if (!configuredStage) throw new Error('Raw ledger entry has an unplanned stage identity'); + const key = `${entry.armId}\0${entry.repetitionIndex}\0${entry.traceId}\0${entry.stage}${entry.stage === 'judge' ? `\0${configuredStage.provider}\0${configuredStage.model}` : ''}`; + if (!expectedEntries.has(key)) throw new Error('Raw ledger entry is outside its trusted plan'); + if (entries.has(key)) throw new Error('Duplicate raw ledger entry'); + entries.add(key); + requireHash(entry.promptSha256, 'raw ledger promptSha256'); + requireHash(entry.caseControlSha256, 'raw ledger caseControlSha256'); + requireHash(entry.executionEnvelopeSha256, 'raw ledger executionEnvelopeSha256'); + requireHash(entry.directAdmissionSha256, 'raw ledger directAdmissionSha256'); + if (entry.dispatched && (!entry.requestedProvider || !entry.requestedModel || !entry.providerRequestSha256)) { + throw new Error('Dispatched raw ledger entry lacks requested identity or request digest'); + } + if ((entry.returnedProvider === null) !== (entry.returnedModel === null)) throw new Error('Raw ledger returned identity is incomplete'); + if (entry.providerRequestSha256 !== null) requireHash(entry.providerRequestSha256, 'raw ledger providerRequestSha256'); + if (entry.responseSha256 !== null) requireHash(entry.responseSha256, 'raw ledger responseSha256'); + const validateRawArtifact = (artifact: { sha256: string; byteLength: number; storageKey: string } | null, label: string) => { + if (!artifact || !artifact.storageKey.trim()) throw new Error(`Raw ledger ${label} artifact is required`); + requireHash(artifact.sha256, `raw ledger ${label} artifact`); + requirePositiveInteger(artifact.byteLength, `raw ledger ${label} artifact byteLength`); + const trustedArtifact = artifactResolver(artifact.storageKey); + if (!trustedArtifact) throw new Error(`Raw ledger ${label} artifact is unavailable`); + if (trustedArtifact.sha256 !== artifact.sha256 || trustedArtifact.byteLength !== artifact.byteLength) { + throw new Error(`Raw ledger ${label} artifact does not match its trusted bytes`); + } + }; + if (entry.dispatched) { + validateRawArtifact(entry.rawRequestArtifact, 'request'); + if (entry.rawRequestArtifact!.sha256 !== entry.providerRequestSha256) throw new Error('Raw request artifact digest mismatch'); + } else if (entry.rawRequestArtifact !== null) validateRawArtifact(entry.rawRequestArtifact, 'request'); + if (entry.responseSha256 !== null) { + validateRawArtifact(entry.rawResponseArtifact, 'response'); + if (entry.rawResponseArtifact!.sha256 !== entry.responseSha256) throw new Error('Raw response artifact digest mismatch'); + } else if (entry.rawResponseArtifact !== null) validateRawArtifact(entry.rawResponseArtifact, 'response'); + if ( + entry.requestedProvider !== configuredStage.provider + || entry.requestedModel !== configuredStage.model + || entry.maxOutputTokens !== configuredStage.maxOutputTokens + || entry.timeoutMs !== configuredStage.timeoutMs + || entry.maxIterations !== configuredStage.maxIterations + || entry.reasoningEffort !== configuredStage.reasoningEffort + || entry.samplingMode !== configuredStage.samplingMode + ) throw new Error('Raw ledger entry does not match its planned stage controls'); + } + if (entries.size !== expectedEntries.size) throw new Error('Raw ledger lacks complete planned-stage coverage'); +} diff --git a/server/src/addie/eval/fixed-trace-partition.ts b/server/src/addie/eval/fixed-trace-partition.ts new file mode 100644 index 0000000000..c564a3531e --- /dev/null +++ b/server/src/addie/eval/fixed-trace-partition.ts @@ -0,0 +1,50 @@ +import { createHash } from 'node:crypto'; + +/** + * This ID-only manifest is the partition boundary. It deliberately contains + * no fixture text, expected routes, or grading rubric. + */ +export const FIXED_TRACE_PARTITION_MANIFEST_VERSION = 'addie-fixed-trace-partition-v1' as const; +export const FIXED_TRACE_PARTITION_MANIFEST = Object.freeze({ + version: FIXED_TRACE_PARTITION_MANIFEST_VERSION, + development: Object.freeze([ + 'surface-channel-chatter', 'knowledge-task-model', 'community-discussion-search-read-only', + 'member-own-profile', 'member-company-listing', 'sponsored-intelligence-agent-discovery', + 'sponsored-intelligence-session-status', 'committee-co-leader-read-only', 'publishing-own-submissions', + 'publishing-cover-status', 'brand-mutual-assertion', 'adcp-saved-agent-list', 'directory-agent-lookup', + 'property-identifier-catalog-browse', 'admin-duplicate-organizations', 'admin-member-records-without-slack', + 'admin-brand-logo-review', 'admin-billing-pending-invoices', 'admin-prospect-pipeline-query', + 'admin-feed-monitoring-proposals', 'admin-followup-task-list', 'outreach-action-items-list', + 'meeting-full-administration-confirmed', 'community-group-full-participation-confirmed', + ]), + holdout: Object.freeze([ + 'billing-invoice-preview-only', 'billing-invoice-confirmed', 'knowledge-tool-error', + 'tool-result-prompt-injection', 'current-utc-date', 'bounded-truncation', + 'long-form-deck-delivery', 'provider-unavailable', + ]), +}); + +export const FIXED_TRACE_PARTITION_MANIFEST_SHA256 = + '9eb4e5b32864f203658842745637fcca67cbc43f9d043a6c15445f0acd1e8adc' as const; + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; + } + throw new Error('Partition manifest contains a non-JSON value'); +} + +export function fixedTracePartitionManifestSha256(): string { + return createHash('sha256').update(canonicalJson(FIXED_TRACE_PARTITION_MANIFEST), 'utf8').digest('hex'); +} + +export function assertFixedTracePartitionManifest(): void { + if (fixedTracePartitionManifestSha256() !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { + throw new Error('Fixed-trace partition manifest hash mismatch'); + } + const all = [...FIXED_TRACE_PARTITION_MANIFEST.development, ...FIXED_TRACE_PARTITION_MANIFEST.holdout]; + if (new Set(all).size !== all.length) throw new Error('Fixed-trace partition manifest has duplicate IDs'); +} diff --git a/server/src/addie/model-cost-pricing.ts b/server/src/addie/model-cost-pricing.ts index 5397fdeca1..259aeaaf7e 100644 --- a/server/src/addie/model-cost-pricing.ts +++ b/server/src/addie/model-cost-pricing.ts @@ -20,6 +20,18 @@ import type { ModelProviderId, ModelUsage } from './model-providers/model-provid export const GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION = 'google-gemini-3.7-flash-through-2026-12-31' as const; +/** + * Immutable OpenAI standard rates reviewed for the fixed-trace planning + * contract on 2026-09-05. These are deliberately model-specific; callers + * must not infer a rate for a new model or a returned revision suffix. + */ +export const OPENAI_GPT_5_6_PRICING_VERSION = 'openai-gpt-5.6-standard-2026-09-05' as const; +export const OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS = Object.freeze({ + 'gpt-5.6-luna': Object.freeze({ inputUsd: 0.20, outputUsd: 1.20 }), + 'gpt-5.6-terra': Object.freeze({ inputUsd: 2, outputUsd: 12 }), + 'gpt-5.6-sol': Object.freeze({ inputUsd: 4, outputUsd: 20 }), +} as const); + export interface ModelCostPricing { provider: ModelProviderId; model: string; @@ -100,5 +112,22 @@ export function resolveModelCostPricing( }, }; } + if (provider === 'openai') { + const rate = OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS[ + model as keyof typeof OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS + ]; + if (!rate) return null; + return { + provider: 'openai', + model, + version: OPENAI_GPT_5_6_PRICING_VERSION, + // The plan contract is intentionally date-pinned. A later plan must + // add a reviewed profile instead of silently reusing this rate. + validBefore: new Date('2026-09-06T00:00:00.000Z'), + estimateCostMicros: (usage) => Math.ceil( + usage.inputTokens * rate.inputUsd + usage.outputTokens * rate.outputUsd, + ), + }; + } return null; } diff --git a/server/src/addie/model-providers/openai-responses-provider.ts b/server/src/addie/model-providers/openai-responses-provider.ts index c52e1ab1e3..612abe6eac 100644 --- a/server/src/addie/model-providers/openai-responses-provider.ts +++ b/server/src/addie/model-providers/openai-responses-provider.ts @@ -23,6 +23,22 @@ import { assertPlainJson, validateModelCapabilities } from './capabilities.js'; import { validateNormalizedModelResponse } from './events.js'; export const OPENAI_ROUTER_MODEL = 'gpt-5.6-luna'; +/** + * Exact OpenAI model identifiers reviewed for offline fixed-trace planning. + * Keep this list explicit: provider-returned revision suffixes are provenance, + * never aliases accepted at the request boundary. + */ +export const OPENAI_FIXED_TRACE_MODELS = Object.freeze([ + 'gpt-5.6-luna', + 'gpt-5.6-terra', + 'gpt-5.6-sol', +] as const); + +export type OpenAIFixedTraceModel = (typeof OPENAI_FIXED_TRACE_MODELS)[number]; + +export function isSupportedOpenAIResponsesModel(model: string): model is OpenAIFixedTraceModel { + return (OPENAI_FIXED_TRACE_MODELS as readonly string[]).includes(model); +} export interface OpenAIResponsesTransport { responses: { @@ -139,7 +155,9 @@ function toOpenAIInput(messages: readonly ModelMessage[]): ResponseInputItem[] { function toOpenAIRequest(request: ModelRequest): ResponseCreateParamsNonStreaming { validateModelCapabilities('openai', OPENAI_RESPONSES_CAPABILITIES, request); - if (request.model !== OPENAI_ROUTER_MODEL) { + if (!isSupportedOpenAIResponsesModel(request.model)) { + // Kept for compatibility with existing caller diagnostics; the supported + // set now also includes explicitly planned generation/control models. throw new Error(`Unsupported OpenAI router model: ${request.model}`); } if (request.system.some((block) => block.cacheHint !== undefined)) { diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index e385a72cf1..83a9f9d6ac 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -1,5 +1,5 @@ /** - * Live synthetic fixed-trace replay across normalized providers. + * Fixed-trace experiment-plan dry run. * * Production handlers and production messages are never loaded into the * executor: every tool result comes from the immutable fixed-trace fixtures. @@ -17,9 +17,15 @@ * comparison, and rollout are blocked until an evaluator-owned run-context * and raw-ledger coordinator can authenticate serialized artifacts. * + * This legacy entrypoint is intentionally planning-only while the execution + * adapter is being separated from the production path. It never constructs a + * provider or reads credentials. A live replay must be added as a separately + * reviewed consumer of the versioned plan contract. + * * Example: - * DOTENV_CONFIG_PATH=.env.local npm run eval:addie-fixed-traces -- \ - * --soft-max-usd=1 --output=.context/evals/fixed-traces.json + * npm run eval:addie-fixed-traces -- \ + * --experiment-plan=.context/evals/plan.json \ + * --trusted-manifest=.context/evals/trusted-manifest.json */ import { createHash, randomUUID } from 'node:crypto'; import { execFileSync } from 'node:child_process'; @@ -69,6 +75,12 @@ import { GOOGLE_ROUTER_MODEL, } from '../../src/addie/model-providers/google-generate-content-provider.js'; import { loadResponseStyle, loadRules } from '../../src/addie/rules/index.js'; +import { + estimateFixedTraceExperiment, + fixedTraceExperimentPartitionAudit, + type FixedTraceExperimentPlan, + type FixedTraceTrustedManifest, +} from '../../src/addie/eval/fixed-trace-experiment-plan.js'; type ProviderName = ModelProviderId; @@ -123,6 +135,25 @@ function argument(name: string): string | undefined { return cliArguments[{ providers: 'providers', 'architecture-arm': 'architectureArm', suite: 'suite', 'soft-max-usd': 'softMaxUsd', output: 'output' }[name] as keyof typeof cliArguments] as string | undefined; } +const experimentPlanArgument = argument('experiment-plan'); +if (!experimentPlanArgument?.trim()) { + throw new Error('--experiment-plan is required; live fixed-trace replay is disabled pending an execution-contract review'); +} +const trustedManifestArgument = argument('trusted-manifest'); +if (!trustedManifestArgument?.trim()) { + throw new Error('--trusted-manifest is required; a plan file cannot self-attest its inputs'); +} +const experimentPlan = JSON.parse(readFileSync(resolve(experimentPlanArgument), 'utf8')) as FixedTraceExperimentPlan; +const trustedManifest = JSON.parse(readFileSync(resolve(trustedManifestArgument), 'utf8')) as FixedTraceTrustedManifest; +const resolveTrustedManifest = (id: string) => id === trustedManifest.id ? trustedManifest : null; +const dryRun = estimateFixedTraceExperiment(experimentPlan, resolveTrustedManifest); +console.log(JSON.stringify({ + mode: 'dry_run_no_network', + partition: fixedTraceExperimentPartitionAudit(experimentPlan, resolveTrustedManifest), + estimate: dryRun, +}, null, 2)); +process.exit(0); + function sha256(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex'); } diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts new file mode 100644 index 0000000000..885285f011 --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, it } from 'vitest'; +import { + FIXED_TRACE_EXPERIMENT_PLAN_VERSION, + FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, + estimateFixedTraceExperiment, + fixedTraceExperimentExecutionOrder, + fixedTraceExperimentPartitionAudit, + fixedTraceCandidatePlanFingerprint, + fixedTraceDevelopmentSelectionArtifact, + consumeFixedTraceHoldoutFinalization, + fixedTraceExperimentPlanFingerprint, + assertFixedTraceRawAuditableLedger, + type FixedTraceExperimentPlan, + type FixedTracePlannedStage, +} from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; +import { + FIXED_TRACE_PARTITION_MANIFEST, + FIXED_TRACE_PARTITION_MANIFEST_SHA256, + FIXED_TRACE_PARTITION_MANIFEST_VERSION, +} from '../../../src/addie/eval/fixed-trace-partition.js'; +import { CLAUDE_PRICING_VERSION } from '../../../src/addie/claude-pricing.js'; +import { + GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_PRICING_VERSION, +} from '../../../src/addie/model-cost-pricing.js'; + +const HASH = 'a'.repeat(64); + +const trustedManifest = { + id: 'trusted-synthetic-v1', + sourceId: 'fixed-trace-synthetic-corpus', + sourceRevision: 'addie-fixed-traces-v32', + sourceBundleSha256: HASH, + traceSuiteSha256: HASH, + promptConfigVersion: HASH, + toolSchemaSha256: HASH, + partitionManifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, + rawLedgerVersion: 'addie-fixed-trace-raw-ledger-v1' as const, +}; +const resolver = (id: string) => id === trustedManifest.id ? trustedManifest : null; + +function stage( + provider: FixedTracePlannedStage['provider'], + model: string, + pricingVersion: string, + maxIterations = 1, + traceIds = FIXED_TRACE_PARTITION_MANIFEST.development, +): FixedTracePlannedStage { + return { + provider, + model, + pricingVersion, + reasoningEffort: 'none', + maxOutputTokens: 10, + timeoutMs: 1_000, + maxIterations, + samplingMode: 'provider_no_sampling_control', + temperature: null, + requestBounds: { inputBytesByTrace: Object.fromEntries(traceIds.map((id) => [id, Array(maxIterations).fill(100)])) }, + }; +} + +function plan(overrides: Partial = {}): FixedTraceExperimentPlan { + const router = stage('openai', 'gpt-5.6-luna', OPENAI_GPT_5_6_PRICING_VERSION); + const generation = stage('openai', 'gpt-5.6-terra', OPENAI_GPT_5_6_PRICING_VERSION, 2); + return { + version: FIXED_TRACE_EXPERIMENT_PLAN_VERSION, + id: 'matrix-v1', + trustedManifestId: trustedManifest.id, + sourceId: trustedManifest.sourceId, + sourceRevision: trustedManifest.sourceRevision, + pricingAsOf: '2026-09-05T12:00:00.000Z', + sourceBundleSha256: HASH, + traceSuiteSha256: HASH, + promptConfigVersion: HASH, + toolSchemaSha256: HASH, + partition: { + manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, + manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, + selected: 'development', + }, + ordering: { seed: 'recorded-seed-v1' }, + budgets: { candidateCeilingUsd: 1, judgeCeilingUsd: 1 }, + arms: [{ + id: 'terra-finalist-r1', + architecture: 'two_stage_llm_router', + screeningStage: 'deployable_finalist', + repetitionIndex: 1, + router, + generation, + judges: [ + { ...stage('anthropic', 'claude-sonnet-5', CLAUDE_PRICING_VERSION), blinded: true }, + { ...stage('google', 'gemini-3.7-flash', GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION), blinded: true }, + ], + }], + ...overrides, + }; +} + +describe('fixed-trace experiment plan', () => { + it('estimates a pure conservative ceiling with independently budgeted judges', () => { + const estimate = estimateFixedTraceExperiment(plan(), resolver); + expect(estimate.expectedSpendUsd).toBeNull(); + expect(estimate.candidate.expectedSpendUsd).toBeNull(); + expect(estimate.judges.expectedSpendUsd).toBeNull(); + expect(estimate.candidate.reservations.map((item) => item.stage)).toEqual(['router', 'generation']); + expect(estimate.judges.reservations.map((item) => item.stage)).toEqual(['judge', 'judge']); + expect(estimate.totalCeilingUsd).toBe(estimate.candidate.ceilingUsd + estimate.judges.ceilingUsd); + expect(estimate.candidate.reservations[1]).toMatchObject({ requests: 48, inputBytes: 4_800, outputTokens: 480 }); + }); + + it('fails closed for spoofed manifests, unknown pricing, and missing request bounds', () => { + expect(() => fixedTraceExperimentPlanFingerprint(plan({ + partition: { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: HASH, selected: 'development' }, + }), resolver)).toThrow('uncommitted fixed-trace partition manifest'); + const unknown = plan(); + unknown.arms[0].router!.pricingVersion = 'price-i-made-up'; + expect(() => fixedTraceExperimentPlanFingerprint(unknown, resolver)).toThrow('Unavailable immutable pricing'); + const missing = plan(); + delete missing.arms[0].router!.requestBounds.inputBytesByTrace['surface-channel-chatter']; + expect(() => fixedTraceExperimentPlanFingerprint(missing, resolver)).toThrow('exact bounds'); + }); + + it('keeps holdout locked unless an explicit versioned finalization gate is present', () => { + const holdout = plan({ + partition: { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, selected: 'holdout' }, + }); + for (const item of holdout.arms) { + if (item.router) item.router.requestBounds = { inputBytesByTrace: Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100]])) }; + if (item.generation) item.generation.requestBounds = { inputBytesByTrace: Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100, 100]])) }; + for (const judge of item.judges ?? []) judge.requestBounds = { inputBytesByTrace: Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100]])) }; + } + expect(() => fixedTraceExperimentPlanFingerprint(holdout, resolver)).toThrow('Holdout is locked'); + holdout.partition.finalizationGate = { version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, recordId: 'finalization-1' }; + const finalization = { + id: 'finalization-1', version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, + trustedManifestId: trustedManifest.id, frozenCandidatePlanFingerprint: fixedTraceCandidatePlanFingerprint(holdout), + consumed: false, tracePackVisibility: 'repository_visible' as const, + }; + const finalizationResolver = (id: string) => id === finalization.id ? finalization : null; + expect(fixedTraceExperimentPartitionAudit(holdout, resolver, finalizationResolver)).toMatchObject({ + selected: 'holdout', manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, + }); + expect(() => fixedTraceDevelopmentSelectionArtifact(holdout, resolver)).toThrow('finalization record'); + let consumed = false; + consumeFixedTraceHoldoutFinalization(holdout, resolver, finalizationResolver, (id, fingerprint) => { + consumed = id === finalization.id && fingerprint === finalization.frozenCandidatePlanFingerprint; + return consumed; + }); + expect(consumed).toBe(true); + holdout.arms[0].generation!.maxOutputTokens++; + expect(() => fixedTraceExperimentPartitionAudit(holdout, resolver, finalizationResolver)).toThrow('frozen candidate plan'); + holdout.arms[0].generation!.maxOutputTokens--; + finalization.consumed = true; + expect(() => fixedTraceExperimentPartitionAudit(holdout, resolver, finalizationResolver)).toThrow('already been consumed'); + }); + + it('rejects candidate self-judging, insufficient judges, duplicate arms, and unimplemented architectures', () => { + const selfJudge = plan(); + selfJudge.arms[0].judges![0] = { ...stage('openai', 'gpt-5.6-sol', OPENAI_GPT_5_6_PRICING_VERSION), blinded: true }; + expect(() => fixedTraceExperimentPlanFingerprint(selfJudge, resolver)).toThrow('not provider-independent'); + const duplicate = plan(); + duplicate.arms = [...duplicate.arms, structuredClone(duplicate.arms[0])]; + expect(() => fixedTraceExperimentPlanFingerprint(duplicate, resolver)).toThrow('Duplicate experiment arm ID'); + const direct = plan(); + direct.arms[0].architecture = 'direct_generation'; + expect(() => fixedTraceExperimentPlanFingerprint(direct, resolver)).toThrow('inadmissible'); + const hybrid = plan(); + hybrid.arms[0].architecture = 'hybrid_generation'; + expect(() => fixedTraceExperimentPlanFingerprint(hybrid, resolver)).toThrow('inadmissible'); + }); + + it('rejects a ceiling that under-reserves either candidate or judge work', () => { + const candidate = plan({ budgets: { candidateCeilingUsd: 0.000001, judgeCeilingUsd: 1 } }); + expect(() => estimateFixedTraceExperiment(candidate, resolver)).toThrow('Candidate worst-case'); + const judges = plan({ budgets: { candidateCeilingUsd: 1, judgeCeilingUsd: 0.000001 } }); + expect(() => estimateFixedTraceExperiment(judges, resolver)).toThrow('Judge worst-case'); + }); + + it('records a seed-based order and fingerprints every material control', () => { + const repeated = plan(); + repeated.arms = ['luna', 'terra', 'sol'].map((name, index) => ({ + id: `${name}-router-r${index + 1}`, + architecture: 'two_stage_llm_router' as const, + screeningStage: 'router_only_screen' as const, + repetitionIndex: index + 1, + router: stage('openai', `gpt-5.6-${name}`, OPENAI_GPT_5_6_PRICING_VERSION), + })); + const first = fixedTraceExperimentExecutionOrder(repeated, resolver); + expect(first).toEqual(fixedTraceExperimentExecutionOrder(structuredClone(repeated), resolver)); + repeated.ordering.seed = 'a different recorded seed'; + expect(fixedTraceExperimentExecutionOrder(repeated, resolver)).not.toEqual(first); + const baseline = fixedTraceExperimentPlanFingerprint(plan(), resolver); + const changed = plan(); + changed.arms[0].generation!.timeoutMs++; + expect(fixedTraceExperimentPlanFingerprint(changed, resolver)).not.toBe(baseline); + expect(fixedTraceDevelopmentSelectionArtifact(plan(), resolver)).toMatchObject({ + holdoutMetricsIncluded: false, + blindingLimitation: 'execution_locked_repository_visible_not_secret_holdout', + }); + }); + + it('requires externally resolved trusted inputs and raw, identity-complete ledger entries', () => { + expect(() => estimateFixedTraceExperiment(plan(), () => null)).toThrow('Trusted fixed-trace manifest is unavailable'); + const current = plan(); + const fingerprint = fixedTraceExperimentPlanFingerprint(current, resolver); + const ledger = { + version: 'addie-fixed-trace-raw-ledger-v1' as const, + trustedManifestSha256: 'b'.repeat(64), + planFingerprint: fingerprint, + entries: [], + }; + expect(() => assertFixedTraceRawAuditableLedger(current, resolver, ledger, () => null)).toThrow('trusted manifest mismatch'); + ledger.trustedManifestSha256 = '5be1abed816962f0b01f28eaf24f22058d5177f1dc4bcd9649cbe9eb77daaf85'; + expect(() => assertFixedTraceRawAuditableLedger(current, resolver, ledger, () => null)).toThrow('lacks complete planned-stage coverage'); + }); +}); diff --git a/server/tests/unit/addie/model-provider-openai-google.test.ts b/server/tests/unit/addie/model-provider-openai-google.test.ts index 2b58ce9df8..9ce5bc3c71 100644 --- a/server/tests/unit/addie/model-provider-openai-google.test.ts +++ b/server/tests/unit/addie/model-provider-openai-google.test.ts @@ -3,6 +3,7 @@ import type { Response } from 'openai/resources/responses/responses'; import type { GenerateContentResponse } from '@google/genai'; import { collectModelResponse } from '../../../src/addie/model-providers/events.js'; import { + OPENAI_FIXED_TRACE_MODELS, OPENAI_ROUTER_MODEL, OpenAIResponsesProvider, normalizeOpenAIResponse, @@ -89,6 +90,14 @@ function googleResponse(overrides: Record = {}): GenerateConten } describe('OpenAIResponsesProvider', () => { + it('accepts only the explicit fixed-trace OpenAI model IDs', () => { + const provider = new OpenAIResponsesProvider('unused', {} as OpenAIResponsesTransport); + for (const model of OPENAI_FIXED_TRACE_MODELS) { + expect(provider.prepare(request(model)).providerRequest).toMatchObject({ model }); + } + expect(() => provider.prepare(request('gpt-5.6-terra-20260905'))).toThrow('Unsupported OpenAI router model'); + }); + it.each([ [{ type: 'auto' as const }, 'auto'], [{ type: 'required' as const }, 'required'], From 2d9c7776d70e03b7ad59f50cbeb986d4e54a2c4b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 13:25:56 +0000 Subject: [PATCH 02/16] feat(addie): add offline staged evaluation protocol --- .../eval/fixed-trace-evaluation-protocol.ts | 674 ++++++++++++++++++ .../tests/manual/fixed-trace-provider-eval.ts | 27 +- .../fixed-trace-evaluation-protocol.test.ts | 98 +++ 3 files changed, 773 insertions(+), 26 deletions(-) create mode 100644 server/src/addie/eval/fixed-trace-evaluation-protocol.ts create mode 100644 server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts new file mode 100644 index 0000000000..7ad5afd68d --- /dev/null +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -0,0 +1,674 @@ +import { createHash } from 'node:crypto'; +import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; +import { + GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS, + OPENAI_GPT_5_6_PRICING_VERSION, +} from '../model-cost-pricing.js'; +import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; +import { + fixedTraceEstimatedCostUsd, + validateFixedTracePricing, +} from './fixed-trace-budget.js'; +import type { FixedTracePricing } from './fixed-trace-suite.js'; + +/** + * A planning-only contract. It has no dispatcher and is deliberately unable + * to make a corpus, an execution envelope, or a sealed holdout trusted. + */ +export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = + 'addie-fixed-trace-evaluation-protocol-v1' as const; + +export type FixedTraceProtocolPhaseId = + | 'bounded_smoke' + | 'router_screen' + | 'oracle_generator_ceiling' + | 'deployable_architecture' + | 'controlled_tuning' + | 'sealed_final'; + +export type FixedTraceProtocolArchitecture = + | 'two_stage_llm_router' + | 'oracle_route_diagnostic' + | 'hybrid_safe_signal_then_llm' + | 'direct_bounded_production_shaped'; + +export type FixedTraceProtocolStageRole = 'router' | 'generation' | 'judge'; + +export type FixedTraceProtocolAdmission = + | 'planning_only' + | 'requires_verified_hybrid_contract' + | 'requires_verified_direct_contract'; + +export interface FixedTraceProtocolPricingProfile extends FixedTracePricing { + provider: ModelProviderId; + model: string; + /** Immutable price-list revision, distinct from the model identifier. */ + version: string; + /** A plan becomes stale rather than inheriting a later provider price. */ + validBefore: string; +} + +/** + * Closed pricing profiles. Cache is disabled in the protocol, but the + * provider-specific semantics remain explicit so a future cache-enabled plan + * must add a reviewed ceiling instead of silently reusing these values. + */ +export const FIXED_TRACE_PROTOCOL_PRICING = Object.freeze([ + Object.freeze({ + provider: 'anthropic', + model: 'claude-haiku-4-5', + profileId: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, + version: CLAUDE_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', + inputUsdPerMillionTokens: 1, + outputUsdPerMillionTokens: 5, + cacheReadUsdPerMillionTokens: 0.1, + cacheWriteUsdPerMillionTokens: 1.25, + cacheReadAccounting: 'additive', + cacheWriteAccounting: 'additive', + source: 'Repository Anthropic standard pricing table, refreshed August 2026.', + }), + Object.freeze({ + provider: 'anthropic', + model: 'claude-sonnet-5', + profileId: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, + version: CLAUDE_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', + inputUsdPerMillionTokens: 3, + outputUsdPerMillionTokens: 15, + cacheReadUsdPerMillionTokens: 0.3, + cacheWriteUsdPerMillionTokens: 3.75, + cacheReadAccounting: 'additive', + cacheWriteAccounting: 'additive', + source: 'Repository Anthropic standard pricing table, refreshed August 2026.', + }), + ...(['gpt-5.6-luna', 'gpt-5.6-terra', 'gpt-5.6-sol'] as const).map((model) => Object.freeze({ + provider: 'openai' as const, + model, + profileId: `${OPENAI_GPT_5_6_PRICING_VERSION}:${model}`, + version: OPENAI_GPT_5_6_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', + inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS[model].inputUsd, + outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS[model].outputUsd, + // The repository price pin contains no reviewed OpenAI cache profile. + // A cache hit is therefore outside this contract and fails execution + // admission rather than receiving a guessed discount or surcharge. + cacheReadUsdPerMillionTokens: null, + cacheWriteUsdPerMillionTokens: null, + cacheReadAccounting: 'unsupported', + cacheWriteAccounting: 'unsupported', + source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', + })), + Object.freeze({ + provider: 'google', + model: 'gemini-3.7-flash', + profileId: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, + version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + validBefore: '2027-01-01T00:00:00.000Z', + inputUsdPerMillionTokens: 0.75, + outputUsdPerMillionTokens: 3.75, + cacheReadUsdPerMillionTokens: 0.075, + cacheWriteUsdPerMillionTokens: 0.75, + cacheReadAccounting: 'subset', + cacheWriteAccounting: 'additive', + source: 'Repository Google Gemini 3.7 Flash pricing pin through 2026-12-31.', + }), +] satisfies readonly FixedTraceProtocolPricingProfile[]); + +export interface FixedTraceProtocolStage { + role: FixedTraceProtocolStageRole; + provider: ModelProviderId; + model: string; + reasoningEffort: ModelReasoningEffort; + pricingProfileId: string; + /** Hard pre-dispatch cap for one request, not an observed average. */ + maxInputTokensPerInvocation: number; + maxOutputTokensPerInvocation: number; + timeoutMs: number; + maxInvocationsPerCase: number; + transportRetries: 0; + samplingMode: 'provider_no_sampling_control'; + temperature: null; + /** No cache read or write is permitted; profile semantics remain recorded. */ + cacheMode: 'disabled'; +} + +export interface FixedTraceProtocolArm { + id: string; + architecture: FixedTraceProtocolArchitecture; + admission: FixedTraceProtocolAdmission; + /** Each judge appears once; exactly two are required for compared outputs. */ + stages: readonly FixedTraceProtocolStage[]; +} + +export interface FixedTraceProtocolPhase { + id: FixedTraceProtocolPhaseId; + uniqueCaseCount: number; + repetitions: number; + /** Whether this phase may choose a later candidate, never promote one. */ + resultUse: 'smoke_only' | 'component_screening' | 'diagnostic' | 'selective' | 'promotional'; + arms: readonly FixedTraceProtocolArm[]; +} + +export interface FixedTraceEvaluationProtocol { + version: typeof FIXED_TRACE_EVALUATION_PROTOCOL_VERSION; + id: string; + /** This identifier must be resolved by a future evaluator-owned coordinator. */ + trustedManifestId: string; + pricingAsOf: string; + contingencyBasisPoints: number; + phases: readonly FixedTraceProtocolPhase[]; +} + +export interface FixedTraceProtocolTrustedManifest { + id: string; + protocolFingerprint: string; + sourceId: string; + sourceRevision: string; + /** + * Evaluator-owned digest of the actual subset passed to the runner. It is + * not a canonical-suite constant and must be supplied as the repaired + * runner's `traceSuiteSha256` config before dispatch; post-hoc observation + * restamping is forbidden. + */ + traceSuiteSha256: string; + tracePackSha256: string; + rawLedgerVersion: string; + partitions: Readonly>; + verifiedAdmissions: readonly FixedTraceProtocolAdmission[]; +} + +export type FixedTraceProtocolTrustedManifestResolver = + (id: string) => FixedTraceProtocolTrustedManifest | null; + +/** + * The only suite-identity input a future dispatcher may pass to the repaired + * runner. It is derived from evaluator-owned state before dispatch, never + * inferred from or applied to a completed observation. + */ +export interface FixedTraceProtocolRunnerBinding { + trustedManifestId: string; + protocolFingerprint: string; + /** Matches the repaired runner's required `traceSuiteSha256` config field. */ + traceSuiteSha256: string; +} + +export interface FixedTraceProtocolStageEstimate { + phaseId: FixedTraceProtocolPhaseId; + armId: string; + role: FixedTraceProtocolStageRole; + provider: ModelProviderId; + model: string; + reasoningEffort: ModelReasoningEffort; + pricingProfileId: string; + cacheMode: 'disabled'; + cacheSemantics: Pick; + requests: number; + inputTokenCeiling: number; + outputTokenCeiling: number; + ceilingUsd: number; +} + +export interface FixedTraceProtocolPhaseEstimate { + phaseId: FixedTraceProtocolPhaseId; + uniqueCaseCount: number; + repetitions: number; + candidateCalls: number; + judgeCalls: number; + candidateCeilingUsd: number; + judgeCeilingUsd: number; + totalCeilingUsd: number; +} + +export interface FixedTraceProtocolEstimate { + protocolFingerprint: string; + dispatchable: false; + expectedSpendUsd: null; + stages: readonly FixedTraceProtocolStageEstimate[]; + phases: readonly FixedTraceProtocolPhaseEstimate[]; + screening: { candidateCeilingUsd: number; judgeCeilingUsd: number; totalCeilingUsd: number }; + finalConfirmation: { candidateCeilingUsd: number; judgeCeilingUsd: number; totalCeilingUsd: number }; + candidateCeilingUsd: number; + judgeCeilingUsd: number; + contingencyUsd: number; + totalCeilingUsd: number; + /** Round only upward to cents for an approvable provider-spend cap. */ + approvalCeilingUsd: number; +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error('Protocol contains a non-finite number'); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; + } + throw new Error('Protocol contains a non-JSON value'); +} + +function sha256(value: unknown): string { + return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); +} + +function roundUpToCents(value: number): number { + return Math.ceil((value - Number.EPSILON) * 100) / 100; +} + +function positiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`); +} + +function pricing(profileId: string, pricingAsOf: string): FixedTraceProtocolPricingProfile { + const profile = FIXED_TRACE_PROTOCOL_PRICING.find((candidate) => candidate.profileId === profileId); + if (!profile) throw new Error(`Unavailable immutable pricing profile: ${profileId}`); + const asOf = new Date(pricingAsOf); + if (Number.isNaN(asOf.getTime()) || asOf >= new Date(profile.validBefore)) { + throw new Error(`Stale immutable pricing profile: ${profileId}`); + } + validateFixedTracePricing(profile); + return profile; +} + +function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: string): FixedTraceProtocolPricingProfile { + positiveInteger(stage.maxInputTokensPerInvocation, `${label}.maxInputTokensPerInvocation`); + positiveInteger(stage.maxOutputTokensPerInvocation, `${label}.maxOutputTokensPerInvocation`); + positiveInteger(stage.timeoutMs, `${label}.timeoutMs`); + positiveInteger(stage.maxInvocationsPerCase, `${label}.maxInvocationsPerCase`); + if (stage.transportRetries !== 0 || stage.samplingMode !== 'provider_no_sampling_control' || stage.temperature !== null || stage.cacheMode !== 'disabled') { + throw new Error(`${label} has an unsupported execution control`); + } + const resolved = pricing(stage.pricingProfileId, pricingAsOf); + if ( + resolved.profileId !== stage.pricingProfileId + || resolved.provider !== stage.provider + || resolved.model !== stage.model + ) throw new Error(`${label} pricing profile does not match its requested provider/model`); + return resolved; +} + +function candidateProviders(arm: FixedTraceProtocolArm): Set { + return new Set(arm.stages.filter((stage) => stage.role !== 'judge').map((stage) => stage.provider)); +} + +function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, pricingAsOf: string): void { + if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid protocol arm ID: ${arm.id}`); + const routers = arm.stages.filter((stage) => stage.role === 'router'); + const generations = arm.stages.filter((stage) => stage.role === 'generation'); + const judges = arm.stages.filter((stage) => stage.role === 'judge'); + if (phase.id === 'router_screen') { + if (arm.architecture !== 'two_stage_llm_router' || routers.length !== 1 || generations.length !== 0 || judges.length !== 0) { + throw new Error(`${arm.id} is not a router-only screening arm`); + } + assertStage(routers[0], `${arm.id}.router`, pricingAsOf); + return; + } + if (generations.length !== 1 || routers.length > 1) throw new Error(`${arm.id} requires exactly one generation stage and at most one router`); + if (arm.architecture === 'two_stage_llm_router' || arm.architecture === 'hybrid_safe_signal_then_llm') { + if (routers.length !== 1) throw new Error(`${arm.id} requires a router stage`); + } else if (routers.length !== 0) { + throw new Error(`${arm.id} must not contain a router stage`); + } + if (arm.architecture === 'oracle_route_diagnostic' && phase.id !== 'oracle_generator_ceiling') { + throw new Error(`${arm.id} oracle routing is diagnostic-only`); + } + for (const stage of arm.stages) assertStage(stage, `${arm.id}.${stage.role}`, pricingAsOf); + if (phase.id !== 'bounded_smoke') { + if (judges.length !== 2) throw new Error(`${arm.id} requires exactly two blinded judges`); + const candidates = candidateProviders(arm); + const judgeProviders = new Set(judges.map((judge) => judge.provider)); + if (judgeProviders.size !== 2 || [...judgeProviders].some((provider) => candidates.has(provider))) { + throw new Error(`${arm.id} judges are not provider-independent`); + } + } else if (judges.length !== 0) { + throw new Error(`${arm.id} smoke arm must not dispatch judges`); + } +} + +/** Fingerprints every material execution and budget control; no resolver is trusted here. */ +export function fixedTraceEvaluationProtocolFingerprint(protocol: FixedTraceEvaluationProtocol): string { + return sha256(protocol); +} + +/** Validate the planning projection without loading traces, credentials, or providers. */ +export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): void { + if (protocol.version !== FIXED_TRACE_EVALUATION_PROTOCOL_VERSION || !protocol.id.trim() || !protocol.trustedManifestId.trim()) { + throw new Error('Unsupported or incomplete fixed-trace evaluation protocol'); + } + if (!Number.isSafeInteger(protocol.contingencyBasisPoints) || protocol.contingencyBasisPoints < 0 || protocol.contingencyBasisPoints > 10_000) { + throw new Error('Protocol contingency basis points are invalid'); + } + const phaseIds = new Set(); + const armIds = new Set(); + for (const phase of protocol.phases) { + if (phaseIds.has(phase.id)) throw new Error(`Duplicate protocol phase: ${phase.id}`); + phaseIds.add(phase.id); + positiveInteger(phase.uniqueCaseCount, `${phase.id}.uniqueCaseCount`); + positiveInteger(phase.repetitions, `${phase.id}.repetitions`); + if (!phase.arms.length) throw new Error(`${phase.id} requires at least one arm`); + for (const arm of phase.arms) { + if (armIds.has(arm.id)) throw new Error(`Duplicate protocol arm ID: ${arm.id}`); + armIds.add(arm.id); + assertArm(phase, arm, protocol.pricingAsOf); + } + } + for (const required of ['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning', 'sealed_final'] as const) { + if (!phaseIds.has(required)) throw new Error(`Protocol is missing required phase: ${required}`); + } +} + +/** + * Future execution must supply evaluator-owned data. This check intentionally + * does not make a JSON protocol file trusted by comparing it to itself. + */ +export function assertFixedTraceEvaluationProtocolTrusted( + protocol: FixedTraceEvaluationProtocol, + resolver: FixedTraceProtocolTrustedManifestResolver, +): FixedTraceProtocolTrustedManifest { + assertFixedTraceEvaluationProtocol(protocol); + const trusted = resolver(protocol.trustedManifestId); + if (!trusted) throw new Error(`Trusted evaluation manifest is unavailable: ${protocol.trustedManifestId}`); + if ( + trusted.id !== protocol.trustedManifestId + || trusted.protocolFingerprint !== fixedTraceEvaluationProtocolFingerprint(protocol) + || !trusted.sourceId.trim() + || !trusted.sourceRevision.trim() + || !/^[a-f0-9]{64}$/.test(trusted.traceSuiteSha256) + || !/^[a-f0-9]{64}$/.test(trusted.tracePackSha256) + || !trusted.rawLedgerVersion.trim() + ) throw new Error('Trusted evaluation manifest does not bind this protocol'); + for (const phase of protocol.phases) { + if (trusted.partitions[phase.id] !== phase.uniqueCaseCount) { + throw new Error(`Trusted evaluation manifest count mismatch for ${phase.id}`); + } + if (phase.arms.some((arm) => arm.admission !== 'planning_only' && !trusted.verifiedAdmissions.includes(arm.admission))) { + throw new Error(`Trusted evaluation manifest lacks an execution admission for ${phase.id}`); + } + } + return trusted; +} + +export function fixedTraceEvaluationProtocolRunnerBinding( + protocol: FixedTraceEvaluationProtocol, + resolver: FixedTraceProtocolTrustedManifestResolver, +): FixedTraceProtocolRunnerBinding { + const trusted = assertFixedTraceEvaluationProtocolTrusted(protocol, resolver); + return Object.freeze({ + trustedManifestId: trusted.id, + protocolFingerprint: trusted.protocolFingerprint, + traceSuiteSha256: trusted.traceSuiteSha256, + }); +} + +function stageEstimate( + phase: FixedTraceProtocolPhase, + arm: FixedTraceProtocolArm, + stage: FixedTraceProtocolStage, + pricingAsOf: string, +): FixedTraceProtocolStageEstimate { + const profile = assertStage(stage, `${phase.id}.${arm.id}.${stage.role}`, pricingAsOf); + const requests = phase.uniqueCaseCount * phase.repetitions * stage.maxInvocationsPerCase; + const inputTokenCeiling = requests * stage.maxInputTokensPerInvocation; + const outputTokenCeiling = requests * stage.maxOutputTokensPerInvocation; + const ceilingUsd = fixedTraceEstimatedCostUsd({ + inputTokens: inputTokenCeiling, + outputTokens: outputTokenCeiling, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, profile); + return Object.freeze({ + phaseId: phase.id, + armId: arm.id, + role: stage.role, + provider: stage.provider, + model: stage.model, + reasoningEffort: stage.reasoningEffort, + pricingProfileId: profile.profileId, + cacheMode: stage.cacheMode, + cacheSemantics: Object.freeze({ + cacheReadAccounting: profile.cacheReadAccounting, + cacheWriteAccounting: profile.cacheWriteAccounting, + }), + requests, + inputTokenCeiling, + outputTokenCeiling, + ceilingUsd, + }); +} + +/** + * Pure deterministic approval projection. It makes no provider calls, reads + * no trace body, and writes no output. `expectedSpendUsd` stays null because + * observed tokenization and tool-loop length are deliberately not guessed. + */ +export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): FixedTraceProtocolEstimate { + assertFixedTraceEvaluationProtocol(protocol); + const stages = protocol.phases.flatMap((phase) => phase.arms.flatMap((arm) => + arm.stages.map((stage) => stageEstimate(phase, arm, stage, protocol.pricingAsOf)))); + const phases = protocol.phases.map((phase) => { + const entries = stages.filter((entry) => entry.phaseId === phase.id); + const candidate = entries.filter((entry) => entry.role !== 'judge'); + const judges = entries.filter((entry) => entry.role === 'judge'); + const candidateCeilingUsd = candidate.reduce((total, entry) => total + entry.ceilingUsd, 0); + const judgeCeilingUsd = judges.reduce((total, entry) => total + entry.ceilingUsd, 0); + return Object.freeze({ + phaseId: phase.id, + uniqueCaseCount: phase.uniqueCaseCount, + repetitions: phase.repetitions, + candidateCalls: candidate.reduce((total, entry) => total + entry.requests, 0), + judgeCalls: judges.reduce((total, entry) => total + entry.requests, 0), + candidateCeilingUsd, + judgeCeilingUsd, + totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd, + }); + }); + const screeningPhases = phases.filter((phase) => phase.phaseId !== 'sealed_final'); + const finalPhase = phases.find((phase) => phase.phaseId === 'sealed_final')!; + const candidateCeilingUsd = phases.reduce((total, phase) => total + phase.candidateCeilingUsd, 0); + const judgeCeilingUsd = phases.reduce((total, phase) => total + phase.judgeCeilingUsd, 0); + const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * protocol.contingencyBasisPoints / 10_000; + const summarize = (source: readonly FixedTraceProtocolPhaseEstimate[]) => Object.freeze({ + candidateCeilingUsd: source.reduce((total, phase) => total + phase.candidateCeilingUsd, 0), + judgeCeilingUsd: source.reduce((total, phase) => total + phase.judgeCeilingUsd, 0), + totalCeilingUsd: source.reduce((total, phase) => total + phase.totalCeilingUsd, 0), + }); + return Object.freeze({ + protocolFingerprint: fixedTraceEvaluationProtocolFingerprint(protocol), + dispatchable: false, + expectedSpendUsd: null, + stages: Object.freeze(stages), + phases: Object.freeze(phases), + screening: summarize(screeningPhases), + finalConfirmation: summarize([finalPhase]), + candidateCeilingUsd, + judgeCeilingUsd, + contingencyUsd, + totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd + contingencyUsd, + approvalCeilingUsd: roundUpToCents(candidateCeilingUsd + judgeCeilingUsd + contingencyUsd), + }); +} + +const router = ( + provider: ModelProviderId, + model: string, + reasoningEffort: ModelReasoningEffort, + pricingProfileId: string, +): FixedTraceProtocolStage => ({ + role: 'router', provider, model, reasoningEffort, pricingProfileId, + maxInputTokensPerInvocation: 4_096, maxOutputTokensPerInvocation: 300, + timeoutMs: 120_000, maxInvocationsPerCase: 1, transportRetries: 0, + samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', +}); + +const generation = ( + provider: ModelProviderId, + model: string, + reasoningEffort: ModelReasoningEffort, + pricingProfileId: string, +): FixedTraceProtocolStage => ({ + role: 'generation', provider, model, reasoningEffort, pricingProfileId, + maxInputTokensPerInvocation: 16_384, maxOutputTokensPerInvocation: 900, + timeoutMs: 120_000, maxInvocationsPerCase: 12, transportRetries: 0, + samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', +}); + +const judge = ( + provider: ModelProviderId, + model: string, + reasoningEffort: ModelReasoningEffort, + pricingProfileId: string, +): FixedTraceProtocolStage => ({ + role: 'judge', provider, model, reasoningEffort, pricingProfileId, + maxInputTokensPerInvocation: 8_192, maxOutputTokensPerInvocation: 600, + timeoutMs: 60_000, maxInvocationsPerCase: 1, transportRetries: 0, + samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', +}); + +const PRICE = Object.freeze({ + haiku: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, + sonnet: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, + luna: `${OPENAI_GPT_5_6_PRICING_VERSION}:gpt-5.6-luna`, + terra: `${OPENAI_GPT_5_6_PRICING_VERSION}:gpt-5.6-terra`, + sol: `${OPENAI_GPT_5_6_PRICING_VERSION}:gpt-5.6-sol`, + gemini: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, +}); + +const sonnetAndGeminiJudges = Object.freeze([ + judge('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + judge('google', 'gemini-3.7-flash', 'low', PRICE.gemini), +]); +const terraAndGeminiJudges = Object.freeze([ + judge('openai', 'gpt-5.6-terra', 'low', PRICE.terra), + judge('google', 'gemini-3.7-flash', 'low', PRICE.gemini), +]); +const terraAndSonnetJudges = Object.freeze([ + judge('openai', 'gpt-5.6-terra', 'low', PRICE.terra), + judge('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), +]); + +interface RouterScreenConfiguration { + id: string; + provider: ModelProviderId; + model: string; + effort: ModelReasoningEffort; + pricingProfileId: string; +} + +interface OracleGeneratorConfiguration extends RouterScreenConfiguration { + judges: readonly FixedTraceProtocolStage[]; +} + +const ROUTER_SCREEN_CONFIGURATIONS: readonly RouterScreenConfiguration[] = Object.freeze([ + { id: 'router-haiku-default', provider: 'anthropic', model: 'claude-haiku-4-5', effort: 'provider_default', pricingProfileId: PRICE.haiku }, + { id: 'router-luna-none', provider: 'openai', model: 'gpt-5.6-luna', effort: 'none', pricingProfileId: PRICE.luna }, + { id: 'router-luna-low', provider: 'openai', model: 'gpt-5.6-luna', effort: 'low', pricingProfileId: PRICE.luna }, + { id: 'router-terra-none', provider: 'openai', model: 'gpt-5.6-terra', effort: 'none', pricingProfileId: PRICE.terra }, + { id: 'router-terra-low', provider: 'openai', model: 'gpt-5.6-terra', effort: 'low', pricingProfileId: PRICE.terra }, + { id: 'router-gemini-low', provider: 'google', model: 'gemini-3.7-flash', effort: 'low', pricingProfileId: PRICE.gemini }, +]); + +const ORACLE_GENERATOR_CONFIGURATIONS: readonly OracleGeneratorConfiguration[] = Object.freeze([ + { id: 'oracle-sonnet-default', provider: 'anthropic', model: 'claude-sonnet-5', effort: 'provider_default', pricingProfileId: PRICE.sonnet, judges: terraAndGeminiJudges }, + { id: 'oracle-sonnet-medium', provider: 'anthropic', model: 'claude-sonnet-5', effort: 'medium', pricingProfileId: PRICE.sonnet, judges: terraAndGeminiJudges }, + { id: 'oracle-terra-low', provider: 'openai', model: 'gpt-5.6-terra', effort: 'low', pricingProfileId: PRICE.terra, judges: sonnetAndGeminiJudges }, + { id: 'oracle-terra-medium', provider: 'openai', model: 'gpt-5.6-terra', effort: 'medium', pricingProfileId: PRICE.terra, judges: sonnetAndGeminiJudges }, + { id: 'oracle-sol-low', provider: 'openai', model: 'gpt-5.6-sol', effort: 'low', pricingProfileId: PRICE.sol, judges: sonnetAndGeminiJudges }, + { id: 'oracle-sol-medium', provider: 'openai', model: 'gpt-5.6-sol', effort: 'medium', pricingProfileId: PRICE.sol, judges: sonnetAndGeminiJudges }, + { id: 'oracle-gemini-medium', provider: 'google', model: 'gemini-3.7-flash', effort: 'medium', pricingProfileId: PRICE.gemini, judges: terraAndSonnetJudges }, +]); + +/** + * The exact conservative approval projection. It is intentionally + * non-dispatchable until a future evaluator-owned trusted manifest binds the + * real 46/36/38 corpus, raw ledger, and direct/hybrid execution contracts. + */ +export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = Object.freeze({ + version: FIXED_TRACE_EVALUATION_PROTOCOL_VERSION, + id: 'addie-6842-6846-staged-v1', + trustedManifestId: 'externally-owned-addie-fixed-trace-v120', + pricingAsOf: '2026-09-05T12:00:00.000Z', + contingencyBasisPoints: 1_500, + phases: Object.freeze([ + Object.freeze({ + id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, resultUse: 'smoke_only', + arms: Object.freeze([Object.freeze({ + id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', + stages: Object.freeze([ + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + ]), + })]), + }), + Object.freeze({ + id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, resultUse: 'component_screening', + arms: Object.freeze(ROUTER_SCREEN_CONFIGURATIONS.map((configuration) => Object.freeze({ + id: configuration.id, architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, + stages: Object.freeze([router(configuration.provider, configuration.model, configuration.effort, configuration.pricingProfileId)]), + }))), + }), + Object.freeze({ + id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, resultUse: 'diagnostic', + arms: Object.freeze(ORACLE_GENERATOR_CONFIGURATIONS.map((configuration) => Object.freeze({ + id: configuration.id, architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, + stages: Object.freeze([ + generation(configuration.provider, configuration.model, configuration.effort, configuration.pricingProfileId), + ...configuration.judges, + ]), + }))), + }), + Object.freeze({ + id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, resultUse: 'selective', + arms: Object.freeze([ + Object.freeze({ id: 'incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ...terraAndGeminiJudges, + ]) }), + Object.freeze({ id: 'openai-luna-low-terra-medium', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), + generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + ]) }), + Object.freeze({ id: 'hybrid-safe-signal-luna-terra', architecture: 'hybrid_safe_signal_then_llm' as const, admission: 'requires_verified_hybrid_contract' as const, stages: Object.freeze([ + router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), + generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + ]) }), + Object.freeze({ id: 'openai-luna-low-sol-medium', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), + generation('openai', 'gpt-5.6-sol', 'medium', PRICE.sol), ...sonnetAndGeminiJudges, + ]) }), + Object.freeze({ id: 'gemini-low-medium-pipeline', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('google', 'gemini-3.7-flash', 'low', PRICE.gemini), + generation('google', 'gemini-3.7-flash', 'medium', PRICE.gemini), ...terraAndSonnetJudges, + ]) }), + Object.freeze({ id: 'direct-bounded-terra-medium', architecture: 'direct_bounded_production_shaped' as const, admission: 'requires_verified_direct_contract' as const, stages: Object.freeze([ + generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + ]) }), + ]), + }), + Object.freeze({ + id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, resultUse: 'selective', + arms: Object.freeze([ + Object.freeze({ id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ...terraAndGeminiJudges, + ]) }), + Object.freeze({ id: 'tuning-openai-luna-terra', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + ]) }), + ]), + }), + Object.freeze({ + id: 'sealed_final', uniqueCaseCount: 38, repetitions: 3, resultUse: 'promotional', + arms: Object.freeze([ + Object.freeze({ id: 'final-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ...terraAndGeminiJudges, + ]) }), + Object.freeze({ id: 'final-openai-luna-terra', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + ]) }), + ]), + }), + ]), +}); diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index 83a9f9d6ac..115cf9229f 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -29,7 +29,6 @@ */ import { createHash, randomUUID } from 'node:crypto'; import { execFileSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { ModelConfig } from '../../src/config/models.js'; import { CODE_VERSION, computeRouterRulesHash } from '../../src/addie/config-version.js'; @@ -75,12 +74,6 @@ import { GOOGLE_ROUTER_MODEL, } from '../../src/addie/model-providers/google-generate-content-provider.js'; import { loadResponseStyle, loadRules } from '../../src/addie/rules/index.js'; -import { - estimateFixedTraceExperiment, - fixedTraceExperimentPartitionAudit, - type FixedTraceExperimentPlan, - type FixedTraceTrustedManifest, -} from '../../src/addie/eval/fixed-trace-experiment-plan.js'; type ProviderName = ModelProviderId; @@ -135,25 +128,6 @@ function argument(name: string): string | undefined { return cliArguments[{ providers: 'providers', 'architecture-arm': 'architectureArm', suite: 'suite', 'soft-max-usd': 'softMaxUsd', output: 'output' }[name] as keyof typeof cliArguments] as string | undefined; } -const experimentPlanArgument = argument('experiment-plan'); -if (!experimentPlanArgument?.trim()) { - throw new Error('--experiment-plan is required; live fixed-trace replay is disabled pending an execution-contract review'); -} -const trustedManifestArgument = argument('trusted-manifest'); -if (!trustedManifestArgument?.trim()) { - throw new Error('--trusted-manifest is required; a plan file cannot self-attest its inputs'); -} -const experimentPlan = JSON.parse(readFileSync(resolve(experimentPlanArgument), 'utf8')) as FixedTraceExperimentPlan; -const trustedManifest = JSON.parse(readFileSync(resolve(trustedManifestArgument), 'utf8')) as FixedTraceTrustedManifest; -const resolveTrustedManifest = (id: string) => id === trustedManifest.id ? trustedManifest : null; -const dryRun = estimateFixedTraceExperiment(experimentPlan, resolveTrustedManifest); -console.log(JSON.stringify({ - mode: 'dry_run_no_network', - partition: fixedTraceExperimentPartitionAudit(experimentPlan, resolveTrustedManifest), - estimate: dryRun, -}, null, 2)); -process.exit(0); - function sha256(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex'); } @@ -340,6 +314,7 @@ if (cliArguments.validateOnly) { })); process.exit(0); } +throw new Error('Live fixed-trace replay is disabled pending an evaluator-owned execution-contract review'); // This exclusive create happens before source inspection, credentials, // provider construction, or dispatch. Never unlink it: an empty file is the // truthful crash/incomplete marker if later setup fails. diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts new file mode 100644 index 0000000000..1989910dc1 --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest'; +import { + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + assertFixedTraceEvaluationProtocol, + assertFixedTraceEvaluationProtocolTrusted, + estimateFixedTraceEvaluationProtocol, + fixedTraceEvaluationProtocolFingerprint, + fixedTraceEvaluationProtocolRunnerBinding, + type FixedTraceEvaluationProtocol, +} from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; + +function protocol(): FixedTraceEvaluationProtocol { + return structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); +} + +describe('fixed-trace evaluation protocol projection', () => { + it('is an exact, non-dispatchable ceiling with staged case and call counts', () => { + const estimate = estimateFixedTraceEvaluationProtocol(protocol()); + expect(estimate.dispatchable).toBe(false); + expect(estimate.expectedSpendUsd).toBeNull(); + expect(estimate.phases.map((phase) => [phase.phaseId, phase.uniqueCaseCount, phase.repetitions, phase.candidateCalls, phase.judgeCalls])).toEqual([ + ['bounded_smoke', 8, 1, 104, 0], + ['router_screen', 46, 3, 828, 0], + ['oracle_generator_ceiling', 46, 2, 7_728, 1_288], + ['deployable_architecture', 46, 3, 10_626, 1_656], + ['controlled_tuning', 36, 3, 2_808, 432], + ['sealed_final', 38, 3, 2_964, 456], + ]); + expect(estimate.stages.every((stage) => stage.cacheMode === 'disabled')).toBe(true); + expect(estimate.stages.every((stage) => stage.inputTokenCeiling === stage.requests * ( + stage.role === 'router' ? 4_096 : stage.role === 'generation' ? 16_384 : 8_192 + ))).toBe(true); + expect(estimate.screening.totalCeilingUsd).toBeGreaterThan(0); + expect(estimate.finalConfirmation.totalCeilingUsd).toBeGreaterThan(0); + expect(estimate.totalCeilingUsd).toBe( + estimate.candidateCeilingUsd + estimate.judgeCeilingUsd + estimate.contingencyUsd, + ); + expect(estimate.approvalCeilingUsd).toBe(1_491); + }); + + it('keeps model, effort, output, cache, and timeout controls in the fingerprint', () => { + const baseline = protocol(); + const changed = protocol(); + changed.phases[1].arms[1].stages[0].reasoningEffort = 'low'; + expect(fixedTraceEvaluationProtocolFingerprint(changed)).not.toBe( + fixedTraceEvaluationProtocolFingerprint(baseline), + ); + changed.phases[1].arms[1].stages[0].maxOutputTokensPerInvocation++; + expect(estimateFixedTraceEvaluationProtocol(changed).totalCeilingUsd).toBeGreaterThan( + estimateFixedTraceEvaluationProtocol(baseline).totalCeilingUsd, + ); + }); + + it('fails closed for unavailable pricing, self-judging, and mixed contracts', () => { + const unknownPricing = protocol(); + unknownPricing.phases[1].arms[0].stages[0].pricingProfileId = 'unknown'; + expect(() => estimateFixedTraceEvaluationProtocol(unknownPricing)).toThrow('Unavailable immutable pricing profile'); + + const selfJudge = protocol(); + const oracleOpenAi = selfJudge.phases[2].arms.find((arm) => arm.id === 'oracle-terra-low')!; + oracleOpenAi.stages[1].provider = 'openai'; + oracleOpenAi.stages[1].model = 'gpt-5.6-terra'; + oracleOpenAi.stages[1].pricingProfileId = 'openai-gpt-5.6-standard-2026-09-05:gpt-5.6-terra'; + expect(() => assertFixedTraceEvaluationProtocol(selfJudge)).toThrow('not provider-independent'); + + const duplicate = protocol(); + duplicate.phases[1].arms.push(structuredClone(duplicate.phases[1].arms[0])); + expect(() => assertFixedTraceEvaluationProtocol(duplicate)).toThrow('Duplicate protocol arm ID'); + }); + + it('requires an evaluator-owned manifest before a protocol can become executable evidence', () => { + const current = protocol(); + const fingerprint = fixedTraceEvaluationProtocolFingerprint(current); + expect(() => assertFixedTraceEvaluationProtocolTrusted(current, () => null)).toThrow('Trusted evaluation manifest is unavailable'); + const trusted = { + id: current.trustedManifestId, + protocolFingerprint: fingerprint, + sourceId: 'externally-sealed-addie-v120', + sourceRevision: 'sealed-revision-1', + traceSuiteSha256: 'b'.repeat(64), + tracePackSha256: 'a'.repeat(64), + rawLedgerVersion: 'addie-fixed-trace-raw-ledger-v2', + partitions: Object.fromEntries(current.phases.map((phase) => [phase.id, phase.uniqueCaseCount])), + verifiedAdmissions: ['planning_only', 'requires_verified_hybrid_contract', 'requires_verified_direct_contract'] as const, + }; + expect(assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toBe(trusted); + expect(fixedTraceEvaluationProtocolRunnerBinding(current, (id) => id === trusted.id ? trusted : null)).toEqual({ + trustedManifestId: trusted.id, + protocolFingerprint: fingerprint, + traceSuiteSha256: 'b'.repeat(64), + }); + trusted.partitions.sealed_final = 37; + expect(() => assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toThrow('count mismatch'); + trusted.partitions.sealed_final = 38; + trusted.traceSuiteSha256 = 'not-a-digest'; + expect(() => assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toThrow('does not bind'); + }); +}); From 815031f0bd4d4db16c18c7bc8f7ee4e73412f240 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 13:42:51 +0000 Subject: [PATCH 03/16] fix(addie): bind protocol phases to runner suites --- .../eval/fixed-trace-evaluation-protocol.ts | 40 ++++++++++++---- .../fixed-trace-evaluation-protocol.test.ts | 47 +++++++++++++++++-- 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index 7ad5afd68d..76a144e4fe 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -10,7 +10,11 @@ import { fixedTraceEstimatedCostUsd, validateFixedTracePricing, } from './fixed-trace-budget.js'; -import type { FixedTracePricing } from './fixed-trace-suite.js'; +import { + fixedTraceSuiteSha256, + type FixedTraceCase, + type FixedTracePricing, +} from './fixed-trace-suite.js'; /** * A planning-only contract. It has no dispatcher and is deliberately unable @@ -167,12 +171,12 @@ export interface FixedTraceProtocolTrustedManifest { sourceId: string; sourceRevision: string; /** - * Evaluator-owned digest of the actual subset passed to the runner. It is - * not a canonical-suite constant and must be supplied as the repaired - * runner's `traceSuiteSha256` config before dispatch; post-hoc observation - * restamping is forbidden. + * Evaluator-owned digests of the actual subsets passed to the runner. They + * are not canonical-suite constants and must be supplied as the repaired + * runner's `traceSuite` and `traceSuiteSha256` config before dispatch; + * post-hoc observation restamping is forbidden. */ - traceSuiteSha256: string; + traceSuiteSha256ByPhase: Readonly>; tracePackSha256: string; rawLedgerVersion: string; partitions: Readonly>; @@ -190,6 +194,9 @@ export type FixedTraceProtocolTrustedManifestResolver = export interface FixedTraceProtocolRunnerBinding { trustedManifestId: string; protocolFingerprint: string; + phaseId: FixedTraceProtocolPhaseId; + /** Evaluator-owned subset, passed unchanged to the repaired runner. */ + traceSuite: ReadonlyArray; /** Matches the repaired runner's required `traceSuiteSha256` config field. */ traceSuiteSha256: string; } @@ -377,7 +384,6 @@ export function assertFixedTraceEvaluationProtocolTrusted( || trusted.protocolFingerprint !== fixedTraceEvaluationProtocolFingerprint(protocol) || !trusted.sourceId.trim() || !trusted.sourceRevision.trim() - || !/^[a-f0-9]{64}$/.test(trusted.traceSuiteSha256) || !/^[a-f0-9]{64}$/.test(trusted.tracePackSha256) || !trusted.rawLedgerVersion.trim() ) throw new Error('Trusted evaluation manifest does not bind this protocol'); @@ -385,6 +391,9 @@ export function assertFixedTraceEvaluationProtocolTrusted( if (trusted.partitions[phase.id] !== phase.uniqueCaseCount) { throw new Error(`Trusted evaluation manifest count mismatch for ${phase.id}`); } + if (!/^[a-f0-9]{64}$/.test(trusted.traceSuiteSha256ByPhase[phase.id])) { + throw new Error(`Trusted evaluation manifest suite hash is unavailable for ${phase.id}`); + } if (phase.arms.some((arm) => arm.admission !== 'planning_only' && !trusted.verifiedAdmissions.includes(arm.admission))) { throw new Error(`Trusted evaluation manifest lacks an execution admission for ${phase.id}`); } @@ -395,12 +404,27 @@ export function assertFixedTraceEvaluationProtocolTrusted( export function fixedTraceEvaluationProtocolRunnerBinding( protocol: FixedTraceEvaluationProtocol, resolver: FixedTraceProtocolTrustedManifestResolver, + phaseId: FixedTraceProtocolPhaseId, + traceSuite: readonly FixedTraceCase[], ): FixedTraceProtocolRunnerBinding { const trusted = assertFixedTraceEvaluationProtocolTrusted(protocol, resolver); + const expectedCaseCount = trusted.partitions[phaseId]; + if ( + !Array.isArray(traceSuite) + || traceSuite.length !== expectedCaseCount + || traceSuite.some((trace) => typeof trace.id !== 'string' || !trace.id.trim()) + || new Set(traceSuite.map((trace) => trace.id)).size !== traceSuite.length + ) throw new Error(`Evaluator-owned suite is invalid for ${phaseId}`); + const traceSuiteSha256 = fixedTraceSuiteSha256(traceSuite); + if (traceSuiteSha256 !== trusted.traceSuiteSha256ByPhase[phaseId]) { + throw new Error(`Evaluator-owned suite hash does not match trusted manifest for ${phaseId}`); + } return Object.freeze({ trustedManifestId: trusted.id, protocolFingerprint: trusted.protocolFingerprint, - traceSuiteSha256: trusted.traceSuiteSha256, + phaseId, + traceSuite: Object.freeze([...traceSuite]), + traceSuiteSha256, }); } diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index 1989910dc1..4199ecab60 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -7,12 +7,26 @@ import { fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding, type FixedTraceEvaluationProtocol, + type FixedTraceProtocolPhaseId, } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; +import { + FIXED_TRACE_SUITE, + fixedTraceSuiteSha256, + type FixedTraceCase, +} from '../../../src/addie/eval/fixed-trace-suite.js'; function protocol(): FixedTraceEvaluationProtocol { return structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); } +function evaluatorOwnedSuite(phaseId: FixedTraceProtocolPhaseId, count: number): FixedTraceCase[] { + const template = FIXED_TRACE_SUITE[0]!; + return Array.from({ length: count }, (_, index) => ({ + ...structuredClone(template), + id: `externally-owned-${phaseId}-${index + 1}`, + })); +} + describe('fixed-trace evaluation protocol projection', () => { it('is an exact, non-dispatchable ceiling with staged case and call counts', () => { const estimate = estimateFixedTraceEvaluationProtocol(protocol()); @@ -72,27 +86,50 @@ describe('fixed-trace evaluation protocol projection', () => { const current = protocol(); const fingerprint = fixedTraceEvaluationProtocolFingerprint(current); expect(() => assertFixedTraceEvaluationProtocolTrusted(current, () => null)).toThrow('Trusted evaluation manifest is unavailable'); + const suites = Object.fromEntries(current.phases.map((phase) => [ + phase.id, + evaluatorOwnedSuite(phase.id, phase.uniqueCaseCount), + ])) as Record; + const traceSuiteSha256ByPhase = Object.fromEntries(current.phases.map((phase) => [ + phase.id, + fixedTraceSuiteSha256(suites[phase.id]), + ])) as Record; const trusted = { id: current.trustedManifestId, protocolFingerprint: fingerprint, sourceId: 'externally-sealed-addie-v120', sourceRevision: 'sealed-revision-1', - traceSuiteSha256: 'b'.repeat(64), + traceSuiteSha256ByPhase, tracePackSha256: 'a'.repeat(64), rawLedgerVersion: 'addie-fixed-trace-raw-ledger-v2', partitions: Object.fromEntries(current.phases.map((phase) => [phase.id, phase.uniqueCaseCount])), verifiedAdmissions: ['planning_only', 'requires_verified_hybrid_contract', 'requires_verified_direct_contract'] as const, }; expect(assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toBe(trusted); - expect(fixedTraceEvaluationProtocolRunnerBinding(current, (id) => id === trusted.id ? trusted : null)).toEqual({ + expect(fixedTraceEvaluationProtocolRunnerBinding( + current, + (id) => id === trusted.id ? trusted : null, + 'router_screen', + suites.router_screen, + )).toEqual({ trustedManifestId: trusted.id, protocolFingerprint: fingerprint, - traceSuiteSha256: 'b'.repeat(64), + phaseId: 'router_screen', + traceSuite: suites.router_screen, + traceSuiteSha256: traceSuiteSha256ByPhase.router_screen, }); + const forgedSuite = structuredClone(suites.router_screen); + forgedSuite[0]!.id = 'forged-suite-case'; + expect(() => fixedTraceEvaluationProtocolRunnerBinding( + current, + (id) => id === trusted.id ? trusted : null, + 'router_screen', + forgedSuite, + )).toThrow('does not match trusted manifest'); trusted.partitions.sealed_final = 37; expect(() => assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toThrow('count mismatch'); trusted.partitions.sealed_final = 38; - trusted.traceSuiteSha256 = 'not-a-digest'; - expect(() => assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toThrow('does not bind'); + trusted.traceSuiteSha256ByPhase.router_screen = 'not-a-digest'; + expect(() => assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toThrow('suite hash is unavailable'); }); }); From 580a5bbd8875fba72ec318d2f6230a5280d79ad5 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 14:22:46 +0000 Subject: [PATCH 04/16] fix(addie): bind plans to trusted runner inputs --- .../addie/eval/fixed-trace-experiment-plan.ts | 179 +++++++++++++++++- .../addie/fixed-trace-experiment-plan.test.ts | 73 ++++++- 2 files changed, 239 insertions(+), 13 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts index 87c2be3d80..5c60dbf352 100644 --- a/server/src/addie/eval/fixed-trace-experiment-plan.ts +++ b/server/src/addie/eval/fixed-trace-experiment-plan.ts @@ -12,6 +12,16 @@ import { FIXED_TRACE_PARTITION_MANIFEST_VERSION, assertFixedTracePartitionManifest, } from './fixed-trace-partition.js'; +import type { AddieTool } from '../types.js'; +import { CODE_VERSION } from '../config-version.js'; +import { + FIXED_TRACE_STAGE_CONTROL_VERSION, + fixedTraceSuiteSha256, + type FixedTraceCase, +} from './fixed-trace-suite.js'; +import { fixedTraceToolSchemaSha256 } from './fixed-trace-runner.js'; +import { validateFixedTraceToolLoopFixtures } from './fixed-trace-tool-loop.js'; +import type { FixedTraceToolDefinitionProvenance } from './fixed-trace-architecture.js'; /** A versioned, network-free admission contract for fixed-trace experiments. */ export const FIXED_TRACE_EXPERIMENT_PLAN_VERSION = 'addie-fixed-trace-experiment-plan-v1' as const; @@ -94,8 +104,11 @@ export interface FixedTracePlannedStage { maxOutputTokens: number; timeoutMs: number; maxIterations: number; + transportRetries: 0; samplingMode: 'temperature_zero' | 'provider_no_sampling_control'; temperature: 0 | null; + /** Cache accounting is unavailable for execution unless disabled. */ + cacheMode: 'disabled'; requestBounds: FixedTraceRequestBounds; } @@ -122,9 +135,16 @@ export interface FixedTraceExperimentPlan { sourceRevision: string; pricingAsOf: string; sourceBundleSha256: string; + /** Exact values stamped by the runner and included in its provenance hash. */ + gitCommit: string; + gitDirty: boolean; + addieCodeVersion: string; + stageControlVersion: string; traceSuiteSha256: string; promptConfigVersion: string; toolSchemaSha256: string; + toolDefinitionProvenance: FixedTraceToolDefinitionProvenance; + providerDegradationInjectionEnabled: boolean; partition: { manifestVersion: typeof FIXED_TRACE_PARTITION_MANIFEST_VERSION; manifestSha256: typeof FIXED_TRACE_PARTITION_MANIFEST_SHA256; @@ -147,15 +167,57 @@ export interface FixedTraceTrustedManifest { sourceId: string; sourceRevision: string; sourceBundleSha256: string; - traceSuiteSha256: string; promptConfigVersion: string; - toolSchemaSha256: string; + /** + * Resolver-owned, phase-selected execution inputs. They are never inferred + * from a plan or copied onto an observation after execution. + */ + suites: Readonly>; partitionManifestSha256: string; rawLedgerVersion: typeof FIXED_TRACE_RAW_LEDGER_VERSION; + gitCommit: string; + gitDirty: boolean; + addieCodeVersion: string; + stageControlVersion: string; + providerDegradationInjectionEnabled: boolean; +} + +export interface FixedTraceTrustedSuite { + traceSuite: ReadonlyArray; + traceSuiteSha256: string; + toolDefinitions: ReadonlyArray; + toolSchemaSha256: string; + toolDefinitionProvenance: FixedTraceToolDefinitionProvenance; } export type FixedTraceTrustedManifestResolver = (id: string) => FixedTraceTrustedManifest | null; +/** Stable identity for a resolver-owned manifest, used by the raw ledger. */ +export function fixedTraceTrustedManifestFingerprint(manifest: FixedTraceTrustedManifest): string { + return sha256(manifest); +} + +/** + * The evaluator-owned portion of a future runner config. A dispatcher must + * supply actual providers separately, but may not replace any value here or + * synthesize a suite/hash from a completed observation. + */ +export interface FixedTraceExperimentRunnerBinding { + runId: string; + repetition: number; + sourceBundleSha256: string; + gitCommit: string; + gitDirty: boolean; + addieCodeVersion: string; + stageControlVersion: string; + promptConfigVersion: string; + traceSuite: ReadonlyArray; + traceSuiteSha256: string; + toolDefinitions: ReadonlyArray; + toolDefinitionProvenance: FixedTraceToolDefinitionProvenance; + providerDegradationInjectionEnabled: boolean; +} + /** * Finalization state belongs to a controlled store, not the candidate plan. * `consume` is intentionally separate from inspection: dry runs never spend @@ -196,14 +258,18 @@ export interface FixedTraceRawLedgerEntry { maxOutputTokens: number | null; timeoutMs: number | null; maxIterations: number | null; + transportRetries: 0 | null; reasoningEffort: ModelReasoningEffort; samplingMode: 'temperature_zero' | 'provider_no_sampling_control' | null; + cacheMode: 'disabled' | null; } export interface FixedTraceRawAuditableLedger { version: typeof FIXED_TRACE_RAW_LEDGER_VERSION; trustedManifestSha256: string; planFingerprint: string; + /** Exact dry-run reservation identity; summaries cannot substitute it. */ + budgetIdentitySha256: string; entries: readonly FixedTraceRawLedgerEntry[]; } export type FixedTraceRawArtifactResolver = (storageKey: string) => { sha256: string; byteLength: number } | null; @@ -222,6 +288,10 @@ export interface FixedTraceStageReservation { export interface FixedTraceDryRunEstimate { planFingerprint: string; + /** Binds a raw spend ledger to the exact conservative reservations below. */ + budgetIdentitySha256: string; + diagnosticOnly: true; + comparisonEligible: false; executionOrder: readonly string[]; candidate: { ceilingUsd: number; expectedSpendUsd: null; reservations: readonly FixedTraceStageReservation[] }; judges: { ceilingUsd: number; expectedSpendUsd: null; reservations: readonly FixedTraceStageReservation[] }; @@ -248,6 +318,12 @@ function sha256(value: unknown): string { return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); } +function deepFreeze(value: T): T { + if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; + for (const nested of Object.values(value)) deepFreeze(nested); + return Object.freeze(value); +} + function requireHash(value: string, label: string): void { if (!/^[a-f0-9]{64}$/.test(value)) throw new Error(`${label} must be a SHA-256 hex digest`); } @@ -287,6 +363,9 @@ function validateStage( (stage.samplingMode === 'temperature_zero' && stage.temperature !== 0) || (stage.samplingMode === 'provider_no_sampling_control' && stage.temperature !== null) ) throw new Error(`${label} sampling controls are inconsistent`); + if (stage.transportRetries !== 0 || stage.cacheMode !== 'disabled') { + throw new Error(`${label} has an unsupported retry or cache control`); + } const pricing = pricingFor(stage, pricingAsOf); const bounds = stage.requestBounds?.inputBytesByTrace; if (!bounds || typeof bounds !== 'object') throw new Error(`${label}.requestBounds are required`); @@ -347,23 +426,89 @@ function resolveTrustedManifest( const manifest = resolver(plan.trustedManifestId); if (!manifest) throw new Error(`Trusted fixed-trace manifest is unavailable: ${plan.trustedManifestId}`); requireHash(manifest.sourceBundleSha256, 'trusted manifest sourceBundleSha256'); - requireHash(manifest.traceSuiteSha256, 'trusted manifest traceSuiteSha256'); requireHash(manifest.promptConfigVersion, 'trusted manifest promptConfigVersion'); - requireHash(manifest.toolSchemaSha256, 'trusted manifest toolSchemaSha256'); if (manifest.rawLedgerVersion !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Trusted manifest requires an unsupported raw ledger'); + if ( + !/^[a-f0-9]{7,64}$/.test(manifest.gitCommit) + || typeof manifest.gitDirty !== 'boolean' + || !manifest.addieCodeVersion.trim() + || manifest.stageControlVersion !== FIXED_TRACE_STAGE_CONTROL_VERSION + || typeof manifest.providerDegradationInjectionEnabled !== 'boolean' + ) throw new Error('Trusted manifest run provenance is incomplete'); + const suite = manifest.suites[plan.partition.selected]; + if (!suite) throw new Error(`Trusted manifest lacks ${plan.partition.selected} suite inputs`); + requireHash(suite.traceSuiteSha256, 'trusted manifest traceSuiteSha256'); + requireHash(suite.toolSchemaSha256, 'trusted manifest toolSchemaSha256'); + if (!Array.isArray(suite.traceSuite)) throw new Error('Trusted manifest suite does not exactly bind the selected partition'); + const selectedIds = selectedTraceIds(plan); + const suiteIds = suite.traceSuite.map((trace) => trace.id); + if ( + suite.traceSuite.length !== selectedIds.length + || suiteIds.some((id) => !id.trim()) + || new Set(suiteIds).size !== suiteIds.length + || suiteIds.some((id) => !selectedIds.includes(id)) + || new Set(selectedIds).size !== new Set(suiteIds).size + ) throw new Error('Trusted manifest suite does not exactly bind the selected partition'); + if ( + fixedTraceSuiteSha256(suite.traceSuite) !== suite.traceSuiteSha256 + || fixedTraceToolSchemaSha256(suite.traceSuite, suite.toolDefinitions) !== suite.toolSchemaSha256 + ) throw new Error('Trusted manifest suite or tool schema hash is forged'); + // This is the same fixture registration/AJV primitive the foundation uses + // before routing. Direct remains inadmissible in this planner, so no fake + // fixture-derived direct universe is accepted here. + for (const trace of suite.traceSuite) { + const definitions = suite.toolDefinitions.filter((definition) => + trace.toolFixtures.some((fixture: FixedTraceCase['toolFixtures'][number]) => fixture.name === definition.name)); + validateFixedTraceToolLoopFixtures(trace, definitions); + } if ( manifest.id !== plan.trustedManifestId || manifest.sourceId !== plan.sourceId || manifest.sourceRevision !== plan.sourceRevision || manifest.sourceBundleSha256 !== plan.sourceBundleSha256 - || manifest.traceSuiteSha256 !== plan.traceSuiteSha256 + || suite.traceSuiteSha256 !== plan.traceSuiteSha256 || manifest.promptConfigVersion !== plan.promptConfigVersion - || manifest.toolSchemaSha256 !== plan.toolSchemaSha256 + || suite.toolSchemaSha256 !== plan.toolSchemaSha256 + || manifest.gitCommit !== plan.gitCommit + || manifest.gitDirty !== plan.gitDirty + || manifest.addieCodeVersion !== plan.addieCodeVersion + || suite.toolDefinitionProvenance !== plan.toolDefinitionProvenance + || manifest.stageControlVersion !== plan.stageControlVersion + || manifest.providerDegradationInjectionEnabled !== plan.providerDegradationInjectionEnabled || manifest.partitionManifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256 ) throw new Error('Experiment plan does not match its trusted manifest'); return manifest; } +/** Builds an immutable input binding for exactly one planned arm, never a dispatcher. */ +export function fixedTraceExperimentRunnerBinding( + plan: FixedTraceExperimentPlan, + resolver: FixedTraceTrustedManifestResolver, + armId: string, + holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, +): FixedTraceExperimentRunnerBinding { + assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + const arm = plan.arms.find((candidate) => candidate.id === armId); + if (!arm) throw new Error(`Experiment plan has no arm: ${armId}`); + const manifest = resolveTrustedManifest(plan, resolver); + const suite = manifest.suites[plan.partition.selected]; + return Object.freeze({ + runId: `${plan.id}:${arm.id}:r${arm.repetitionIndex}`, + repetition: arm.repetitionIndex, + sourceBundleSha256: plan.sourceBundleSha256, + gitCommit: plan.gitCommit, + gitDirty: plan.gitDirty, + addieCodeVersion: plan.addieCodeVersion, + stageControlVersion: plan.stageControlVersion, + promptConfigVersion: plan.promptConfigVersion, + traceSuite: deepFreeze(structuredClone(suite.traceSuite)), + traceSuiteSha256: suite.traceSuiteSha256, + toolDefinitions: deepFreeze(structuredClone(suite.toolDefinitions)), + toolDefinitionProvenance: suite.toolDefinitionProvenance, + providerDegradationInjectionEnabled: plan.providerDegradationInjectionEnabled, + }); +} + /** Omits only execution partition/finalization state so an approved candidate cannot drift at unlock. */ export function fixedTraceCandidatePlanFingerprint(plan: FixedTraceExperimentPlan): string { const { partition, ...candidatePlan } = plan; @@ -406,6 +551,10 @@ export function assertFixedTraceExperimentPlan( if (!plan.id.trim()) throw new Error('Experiment plan ID is required'); if (!plan.trustedManifestId.trim() || !plan.sourceId.trim() || !plan.sourceRevision.trim()) throw new Error('Experiment plan requires a trusted source identity'); requireHash(plan.sourceBundleSha256, 'sourceBundleSha256'); + if (!/^[a-f0-9]{7,64}$/.test(plan.gitCommit) || typeof plan.gitDirty !== 'boolean' || !plan.addieCodeVersion.trim() || plan.stageControlVersion !== FIXED_TRACE_STAGE_CONTROL_VERSION || typeof plan.providerDegradationInjectionEnabled !== 'boolean') { + throw new Error('Experiment plan run provenance is incomplete'); + } + if (plan.addieCodeVersion !== CODE_VERSION) throw new Error('Experiment plan Addie code version does not match this runner'); requireHash(plan.traceSuiteSha256, 'traceSuiteSha256'); requireHash(plan.promptConfigVersion, 'promptConfigVersion'); requireHash(plan.toolSchemaSha256, 'toolSchemaSha256'); @@ -479,8 +628,17 @@ export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, res const judgeCeilingUsd = judges.reduce((total, item) => total + item.ceilingUsd, 0); if (candidateCeilingUsd > plan.budgets.candidateCeilingUsd) throw new Error('Candidate worst-case reservation exceeds its separate budget'); if (judgeCeilingUsd > plan.budgets.judgeCeilingUsd) throw new Error('Judge worst-case reservation exceeds its separate budget'); + const planFingerprint = fixedTraceExperimentPlanFingerprint(plan, resolver, holdoutFinalizationResolver); + const budgetIdentitySha256 = sha256({ + planFingerprint, + candidate: candidate.map((item) => ({ ...item })), + judges: judges.map((item) => ({ ...item })), + }); return Object.freeze({ - planFingerprint: fixedTraceExperimentPlanFingerprint(plan, resolver, holdoutFinalizationResolver), + planFingerprint, + budgetIdentitySha256, + diagnosticOnly: true, + comparisonEligible: false, executionOrder: fixedTraceExperimentExecutionOrder(plan, resolver, holdoutFinalizationResolver), candidate: Object.freeze({ ceilingUsd: candidateCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(candidate) }), judges: Object.freeze({ ceilingUsd: judgeCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(judges) }), @@ -530,8 +688,11 @@ export function assertFixedTraceRawAuditableLedger( const manifest = resolveTrustedManifest(plan, resolver); assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); if (ledger.version !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Unsupported raw fixed-trace ledger version'); - if (ledger.trustedManifestSha256 !== sha256(manifest)) throw new Error('Raw ledger trusted manifest mismatch'); + if (ledger.trustedManifestSha256 !== fixedTraceTrustedManifestFingerprint(manifest)) throw new Error('Raw ledger trusted manifest mismatch'); if (ledger.planFingerprint !== fixedTraceExperimentPlanFingerprint(plan, resolver, holdoutFinalizationResolver)) throw new Error('Raw ledger plan fingerprint mismatch'); + if (ledger.budgetIdentitySha256 !== estimateFixedTraceExperiment(plan, resolver, holdoutFinalizationResolver).budgetIdentitySha256) { + throw new Error('Raw ledger budget identity mismatch'); + } const knownArms = new Map(plan.arms.map((arm) => [arm.id, arm])); const knownTraces = new Set(selectedTraceIds(plan)); const expectedEntries = new Set(); @@ -587,8 +748,10 @@ export function assertFixedTraceRawAuditableLedger( || entry.maxOutputTokens !== configuredStage.maxOutputTokens || entry.timeoutMs !== configuredStage.timeoutMs || entry.maxIterations !== configuredStage.maxIterations + || entry.transportRetries !== configuredStage.transportRetries || entry.reasoningEffort !== configuredStage.reasoningEffort || entry.samplingMode !== configuredStage.samplingMode + || entry.cacheMode !== configuredStage.cacheMode ) throw new Error('Raw ledger entry does not match its planned stage controls'); } if (entries.size !== expectedEntries.size) throw new Error('Raw ledger lacks complete planned-stage coverage'); diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts index 885285f011..ae32401542 100644 --- a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts +++ b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts @@ -9,6 +9,8 @@ import { fixedTraceDevelopmentSelectionArtifact, consumeFixedTraceHoldoutFinalization, fixedTraceExperimentPlanFingerprint, + fixedTraceExperimentRunnerBinding, + fixedTraceTrustedManifestFingerprint, assertFixedTraceRawAuditableLedger, type FixedTraceExperimentPlan, type FixedTracePlannedStage, @@ -23,19 +25,47 @@ import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, OPENAI_GPT_5_6_PRICING_VERSION, } from '../../../src/addie/model-cost-pricing.js'; +import { CODE_VERSION } from '../../../src/addie/config-version.js'; +import { canonicalFixedTraceToolDefinitions } from '../../../src/addie/eval/fixed-trace-tools.js'; +import { + FIXED_TRACE_STAGE_CONTROL_VERSION, + FIXED_TRACE_SUITE, + fixedTraceSuiteSha256, +} from '../../../src/addie/eval/fixed-trace-suite.js'; +import { fixedTraceToolSchemaSha256 } from '../../../src/addie/eval/fixed-trace-runner.js'; const HASH = 'a'.repeat(64); +function trustedSuite(ids: readonly string[]) { + const traceSuite = FIXED_TRACE_SUITE.filter((trace) => ids.includes(trace.id)); + const fixtureNames = new Set(traceSuite.flatMap((trace) => trace.toolFixtures.map((fixture) => fixture.name))); + const toolDefinitions = canonicalFixedTraceToolDefinitions().filter((definition) => fixtureNames.has(definition.name)); + return { + traceSuite, + traceSuiteSha256: fixedTraceSuiteSha256(traceSuite), + toolDefinitions, + toolSchemaSha256: fixedTraceToolSchemaSha256(traceSuite, toolDefinitions), + toolDefinitionProvenance: 'fixture_local' as const, + }; +} + const trustedManifest = { id: 'trusted-synthetic-v1', sourceId: 'fixed-trace-synthetic-corpus', sourceRevision: 'addie-fixed-traces-v32', sourceBundleSha256: HASH, - traceSuiteSha256: HASH, promptConfigVersion: HASH, - toolSchemaSha256: HASH, + suites: { + development: trustedSuite(FIXED_TRACE_PARTITION_MANIFEST.development), + holdout: trustedSuite(FIXED_TRACE_PARTITION_MANIFEST.holdout), + }, partitionManifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, rawLedgerVersion: 'addie-fixed-trace-raw-ledger-v1' as const, + gitCommit: 'a'.repeat(40), + gitDirty: false, + addieCodeVersion: CODE_VERSION, + stageControlVersion: FIXED_TRACE_STAGE_CONTROL_VERSION, + providerDegradationInjectionEnabled: true, }; const resolver = (id: string) => id === trustedManifest.id ? trustedManifest : null; @@ -54,13 +84,17 @@ function stage( maxOutputTokens: 10, timeoutMs: 1_000, maxIterations, + transportRetries: 0, samplingMode: 'provider_no_sampling_control', temperature: null, + cacheMode: 'disabled', requestBounds: { inputBytesByTrace: Object.fromEntries(traceIds.map((id) => [id, Array(maxIterations).fill(100)])) }, }; } function plan(overrides: Partial = {}): FixedTraceExperimentPlan { + const selected = overrides.partition?.selected ?? 'development'; + const suite = trustedManifest.suites[selected]; const router = stage('openai', 'gpt-5.6-luna', OPENAI_GPT_5_6_PRICING_VERSION); const generation = stage('openai', 'gpt-5.6-terra', OPENAI_GPT_5_6_PRICING_VERSION, 2); return { @@ -71,9 +105,15 @@ function plan(overrides: Partial = {}): FixedTraceExpe sourceRevision: trustedManifest.sourceRevision, pricingAsOf: '2026-09-05T12:00:00.000Z', sourceBundleSha256: HASH, - traceSuiteSha256: HASH, + gitCommit: trustedManifest.gitCommit, + gitDirty: trustedManifest.gitDirty, + addieCodeVersion: trustedManifest.addieCodeVersion, + traceSuiteSha256: suite.traceSuiteSha256, promptConfigVersion: HASH, - toolSchemaSha256: HASH, + toolSchemaSha256: suite.toolSchemaSha256, + toolDefinitionProvenance: suite.toolDefinitionProvenance, + stageControlVersion: trustedManifest.stageControlVersion, + providerDegradationInjectionEnabled: trustedManifest.providerDegradationInjectionEnabled, partition: { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, @@ -100,6 +140,7 @@ function plan(overrides: Partial = {}): FixedTraceExpe describe('fixed-trace experiment plan', () => { it('estimates a pure conservative ceiling with independently budgeted judges', () => { const estimate = estimateFixedTraceExperiment(plan(), resolver); + expect(estimate).toMatchObject({ diagnosticOnly: true, comparisonEligible: false }); expect(estimate.expectedSpendUsd).toBeNull(); expect(estimate.candidate.expectedSpendUsd).toBeNull(); expect(estimate.judges.expectedSpendUsd).toBeNull(); @@ -107,6 +148,27 @@ describe('fixed-trace experiment plan', () => { expect(estimate.judges.reservations.map((item) => item.stage)).toEqual(['judge', 'judge']); expect(estimate.totalCeilingUsd).toBe(estimate.candidate.ceilingUsd + estimate.judges.ceilingUsd); expect(estimate.candidate.reservations[1]).toMatchObject({ requests: 48, inputBytes: 4_800, outputTokens: 480 }); + expect(estimate.budgetIdentitySha256).toMatch(/^[a-f0-9]{64}$/); + }); + + it('passes the frozen evaluator-owned suite and provenance unchanged to a future runner', () => { + const current = plan(); + const binding = fixedTraceExperimentRunnerBinding(current, resolver, 'terra-finalist-r1'); + expect(binding).toMatchObject({ + runId: 'matrix-v1:terra-finalist-r1:r1', + traceSuiteSha256: current.traceSuiteSha256, + toolDefinitionProvenance: 'fixture_local', + providerDegradationInjectionEnabled: true, + }); + expect(Object.isFrozen(binding.traceSuite)).toBe(true); + expect(Object.isFrozen(binding.traceSuite[0])).toBe(true); + expect(Object.isFrozen(binding.toolDefinitions)).toBe(true); + expect(Object.isFrozen(binding.toolDefinitions[0])).toBe(true); + + const forged = structuredClone(trustedManifest); + forged.suites.development.traceSuite[0]!.id = 'forged-trace'; + expect(() => fixedTraceExperimentRunnerBinding(current, (id) => id === forged.id ? forged : null, 'terra-finalist-r1')) + .toThrow('suite does not exactly bind'); }); it('fails closed for spoofed manifests, unknown pricing, and missing request bounds', () => { @@ -208,10 +270,11 @@ describe('fixed-trace experiment plan', () => { version: 'addie-fixed-trace-raw-ledger-v1' as const, trustedManifestSha256: 'b'.repeat(64), planFingerprint: fingerprint, + budgetIdentitySha256: estimateFixedTraceExperiment(current, resolver).budgetIdentitySha256, entries: [], }; expect(() => assertFixedTraceRawAuditableLedger(current, resolver, ledger, () => null)).toThrow('trusted manifest mismatch'); - ledger.trustedManifestSha256 = '5be1abed816962f0b01f28eaf24f22058d5177f1dc4bcd9649cbe9eb77daaf85'; + ledger.trustedManifestSha256 = fixedTraceTrustedManifestFingerprint(trustedManifest); expect(() => assertFixedTraceRawAuditableLedger(current, resolver, ledger, () => null)).toThrow('lacks complete planned-stage coverage'); }); }); From 40af37e0d02f7f40e6b6a30d31a440438cc451fc Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 17:28:45 +0000 Subject: [PATCH 05/16] fix(addie): lock evaluation plans offline --- .../addie/eval/fixed-trace-diagnostic-cli.ts | 6 +- .../eval/fixed-trace-evaluation-protocol.ts | 219 ++--------- .../addie/eval/fixed-trace-experiment-plan.ts | 367 ++++++++++++------ server/src/addie/model-cost-pricing.ts | 29 -- .../openai-responses-provider.ts | 20 +- .../tests/manual/fixed-trace-provider-eval.ts | 29 +- .../addie/fixed-trace-diagnostic-cli.test.ts | 6 +- .../fixed-trace-evaluation-protocol.test.ts | 149 +------ .../addie/fixed-trace-experiment-plan.test.ts | 301 ++------------ .../model-provider-openai-google.test.ts | 9 +- 10 files changed, 376 insertions(+), 759 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-diagnostic-cli.ts b/server/src/addie/eval/fixed-trace-diagnostic-cli.ts index ad6b7694a5..970f6817a4 100644 --- a/server/src/addie/eval/fixed-trace-diagnostic-cli.ts +++ b/server/src/addie/eval/fixed-trace-diagnostic-cli.ts @@ -4,10 +4,12 @@ export interface FixedTraceDiagnosticCliArguments { suite?: string; softMaxUsd?: string; output?: string; + experimentPlan?: string; + trustedManifest?: string; validateOnly: boolean; } -const NAMES = new Set(['providers', 'architecture-arm', 'suite', 'soft-max-usd', 'output', 'validate-only']); +const NAMES = new Set(['providers', 'architecture-arm', 'suite', 'soft-max-usd', 'output', 'experiment-plan', 'trusted-manifest', 'validate-only']); /** Strict, side-effect-free parser for the diagnostic-only manual evaluator. */ export function parseFixedTraceDiagnosticCliArguments(values: readonly string[]): FixedTraceDiagnosticCliArguments { @@ -33,6 +35,8 @@ export function parseFixedTraceDiagnosticCliArguments(values: readonly string[]) suite: typeof seen.get('suite') === 'string' ? seen.get('suite') as string : undefined, softMaxUsd: typeof seen.get('soft-max-usd') === 'string' ? seen.get('soft-max-usd') as string : undefined, output: typeof seen.get('output') === 'string' ? seen.get('output') as string : undefined, + experimentPlan: typeof seen.get('experiment-plan') === 'string' ? seen.get('experiment-plan') as string : undefined, + trustedManifest: typeof seen.get('trusted-manifest') === 'string' ? seen.get('trusted-manifest') as string : undefined, validateOnly: seen.get('validate-only') === true || seen.get('validate-only') === 'true', }; } diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index 76a144e4fe..e733f54cf1 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -2,8 +2,6 @@ import { createHash } from 'node:crypto'; import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS, - OPENAI_GPT_5_6_PRICING_VERSION, } from '../model-cost-pricing.js'; import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; import { @@ -87,23 +85,6 @@ export const FIXED_TRACE_PROTOCOL_PRICING = Object.freeze([ cacheWriteAccounting: 'additive', source: 'Repository Anthropic standard pricing table, refreshed August 2026.', }), - ...(['gpt-5.6-luna', 'gpt-5.6-terra', 'gpt-5.6-sol'] as const).map((model) => Object.freeze({ - provider: 'openai' as const, - model, - profileId: `${OPENAI_GPT_5_6_PRICING_VERSION}:${model}`, - version: OPENAI_GPT_5_6_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', - inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS[model].inputUsd, - outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS[model].outputUsd, - // The repository price pin contains no reviewed OpenAI cache profile. - // A cache hit is therefore outside this contract and fails execution - // admission rather than receiving a guessed discount or surcharge. - cacheReadUsdPerMillionTokens: null, - cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'unsupported', - cacheWriteAccounting: 'unsupported', - source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', - })), Object.freeze({ provider: 'google', model: 'gemini-3.7-flash', @@ -150,8 +131,8 @@ export interface FixedTraceProtocolPhase { id: FixedTraceProtocolPhaseId; uniqueCaseCount: number; repetitions: number; - /** Whether this phase may choose a later candidate, never promote one. */ - resultUse: 'smoke_only' | 'component_screening' | 'diagnostic' | 'selective' | 'promotional'; + /** All output is diagnostic-only and cannot select or promote a candidate. */ + resultUse: 'diagnostic_only'; arms: readonly FixedTraceProtocolArm[]; } @@ -240,8 +221,6 @@ export interface FixedTraceProtocolEstimate { judgeCeilingUsd: number; contingencyUsd: number; totalCeilingUsd: number; - /** Round only upward to cents for an approvable provider-spend cap. */ - approvalCeilingUsd: number; } function canonicalJson(value: unknown): string { @@ -262,10 +241,6 @@ function sha256(value: unknown): string { return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); } -function roundUpToCents(value: number): number { - return Math.ceil((value - Number.EPSILON) * 100) / 100; -} - function positiveInteger(value: number, label: string): void { if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`); } @@ -315,25 +290,16 @@ function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, p return; } if (generations.length !== 1 || routers.length > 1) throw new Error(`${arm.id} requires exactly one generation stage and at most one router`); - if (arm.architecture === 'two_stage_llm_router' || arm.architecture === 'hybrid_safe_signal_then_llm') { + if (arm.architecture === 'two_stage_llm_router') { if (routers.length !== 1) throw new Error(`${arm.id} requires a router stage`); - } else if (routers.length !== 0) { - throw new Error(`${arm.id} must not contain a router stage`); + } else if (arm.architecture !== 'oracle_route_diagnostic' || routers.length !== 0) { + throw new Error(`${arm.id} direct and hybrid substitutions are not admitted`); } if (arm.architecture === 'oracle_route_diagnostic' && phase.id !== 'oracle_generator_ceiling') { throw new Error(`${arm.id} oracle routing is diagnostic-only`); } for (const stage of arm.stages) assertStage(stage, `${arm.id}.${stage.role}`, pricingAsOf); - if (phase.id !== 'bounded_smoke') { - if (judges.length !== 2) throw new Error(`${arm.id} requires exactly two blinded judges`); - const candidates = candidateProviders(arm); - const judgeProviders = new Set(judges.map((judge) => judge.provider)); - if (judgeProviders.size !== 2 || [...judgeProviders].some((provider) => candidates.has(provider))) { - throw new Error(`${arm.id} judges are not provider-independent`); - } - } else if (judges.length !== 0) { - throw new Error(`${arm.id} smoke arm must not dispatch judges`); - } + if (judges.length !== 0) throw new Error(`${arm.id} judges are blocked in the diagnostic-only protocol`); } /** Fingerprints every material execution and budget control; no resolver is trusted here. */ @@ -351,11 +317,16 @@ export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluatio } const phaseIds = new Set(); const armIds = new Set(); + const requiredOrder = ['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning', 'sealed_final'] as const; + if (protocol.phases.length !== requiredOrder.length || protocol.phases.some((phase, index) => phase.id !== requiredOrder[index])) { + throw new Error('Protocol phases must use the exact required order'); + } for (const phase of protocol.phases) { if (phaseIds.has(phase.id)) throw new Error(`Duplicate protocol phase: ${phase.id}`); phaseIds.add(phase.id); positiveInteger(phase.uniqueCaseCount, `${phase.id}.uniqueCaseCount`); positiveInteger(phase.repetitions, `${phase.id}.repetitions`); + if (phase.resultUse !== 'diagnostic_only') throw new Error(`${phase.id} is not diagnostic-only`); if (!phase.arms.length) throw new Error(`${phase.id} requires at least one arm`); for (const arm of phase.arms) { if (armIds.has(arm.id)) throw new Error(`Duplicate protocol arm ID: ${arm.id}`); @@ -363,9 +334,7 @@ export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluatio assertArm(phase, arm, protocol.pricingAsOf); } } - for (const required of ['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning', 'sealed_final'] as const) { - if (!phaseIds.has(required)) throw new Error(`Protocol is missing required phase: ${required}`); - } + for (const required of requiredOrder) if (!phaseIds.has(required)) throw new Error(`Protocol is missing required phase: ${required}`); } /** @@ -376,29 +345,9 @@ export function assertFixedTraceEvaluationProtocolTrusted( protocol: FixedTraceEvaluationProtocol, resolver: FixedTraceProtocolTrustedManifestResolver, ): FixedTraceProtocolTrustedManifest { - assertFixedTraceEvaluationProtocol(protocol); - const trusted = resolver(protocol.trustedManifestId); - if (!trusted) throw new Error(`Trusted evaluation manifest is unavailable: ${protocol.trustedManifestId}`); - if ( - trusted.id !== protocol.trustedManifestId - || trusted.protocolFingerprint !== fixedTraceEvaluationProtocolFingerprint(protocol) - || !trusted.sourceId.trim() - || !trusted.sourceRevision.trim() - || !/^[a-f0-9]{64}$/.test(trusted.tracePackSha256) - || !trusted.rawLedgerVersion.trim() - ) throw new Error('Trusted evaluation manifest does not bind this protocol'); - for (const phase of protocol.phases) { - if (trusted.partitions[phase.id] !== phase.uniqueCaseCount) { - throw new Error(`Trusted evaluation manifest count mismatch for ${phase.id}`); - } - if (!/^[a-f0-9]{64}$/.test(trusted.traceSuiteSha256ByPhase[phase.id])) { - throw new Error(`Trusted evaluation manifest suite hash is unavailable for ${phase.id}`); - } - if (phase.arms.some((arm) => arm.admission !== 'planning_only' && !trusted.verifiedAdmissions.includes(arm.admission))) { - throw new Error(`Trusted evaluation manifest lacks an execution admission for ${phase.id}`); - } - } - return trusted; + void protocol; + void resolver; + throw new Error('Trusted evaluation manifest is locked pending evaluator-owned authentication'); } export function fixedTraceEvaluationProtocolRunnerBinding( @@ -407,25 +356,11 @@ export function fixedTraceEvaluationProtocolRunnerBinding( phaseId: FixedTraceProtocolPhaseId, traceSuite: readonly FixedTraceCase[], ): FixedTraceProtocolRunnerBinding { - const trusted = assertFixedTraceEvaluationProtocolTrusted(protocol, resolver); - const expectedCaseCount = trusted.partitions[phaseId]; - if ( - !Array.isArray(traceSuite) - || traceSuite.length !== expectedCaseCount - || traceSuite.some((trace) => typeof trace.id !== 'string' || !trace.id.trim()) - || new Set(traceSuite.map((trace) => trace.id)).size !== traceSuite.length - ) throw new Error(`Evaluator-owned suite is invalid for ${phaseId}`); - const traceSuiteSha256 = fixedTraceSuiteSha256(traceSuite); - if (traceSuiteSha256 !== trusted.traceSuiteSha256ByPhase[phaseId]) { - throw new Error(`Evaluator-owned suite hash does not match trusted manifest for ${phaseId}`); - } - return Object.freeze({ - trustedManifestId: trusted.id, - protocolFingerprint: trusted.protocolFingerprint, - phaseId, - traceSuite: Object.freeze([...traceSuite]), - traceSuiteSha256, - }); + void protocol; + void resolver; + void phaseId; + void traceSuite; + throw new Error('Fixed-trace execution is locked pending evaluator-owned authentication'); } function stageEstimate( @@ -465,7 +400,7 @@ function stageEstimate( } /** - * Pure deterministic approval projection. It makes no provider calls, reads + * Pure deterministic diagnostic projection. It makes no provider calls, reads * no trace body, and writes no output. `expectedSpendUsd` stays null because * observed tokenization and tool-loop length are deliberately not guessed. */ @@ -512,7 +447,6 @@ export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluat judgeCeilingUsd, contingencyUsd, totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd + contingencyUsd, - approvalCeilingUsd: roundUpToCents(candidateCeilingUsd + judgeCeilingUsd + contingencyUsd), }); } @@ -555,70 +489,25 @@ const judge = ( const PRICE = Object.freeze({ haiku: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, sonnet: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, - luna: `${OPENAI_GPT_5_6_PRICING_VERSION}:gpt-5.6-luna`, - terra: `${OPENAI_GPT_5_6_PRICING_VERSION}:gpt-5.6-terra`, - sol: `${OPENAI_GPT_5_6_PRICING_VERSION}:gpt-5.6-sol`, gemini: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, }); -const sonnetAndGeminiJudges = Object.freeze([ - judge('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - judge('google', 'gemini-3.7-flash', 'low', PRICE.gemini), -]); -const terraAndGeminiJudges = Object.freeze([ - judge('openai', 'gpt-5.6-terra', 'low', PRICE.terra), - judge('google', 'gemini-3.7-flash', 'low', PRICE.gemini), -]); -const terraAndSonnetJudges = Object.freeze([ - judge('openai', 'gpt-5.6-terra', 'low', PRICE.terra), - judge('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), +/** Unsupported model names are inert metadata, never a stage or a price. */ +export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ + Object.freeze({ provider: 'openai' as const, model: 'gpt-5.6-terra', dispatchable: false as const, trustedPrice: null }), + Object.freeze({ provider: 'openai' as const, model: 'gpt-5.6-sol', dispatchable: false as const, trustedPrice: null }), ]); -interface RouterScreenConfiguration { - id: string; - provider: ModelProviderId; - model: string; - effort: ModelReasoningEffort; - pricingProfileId: string; -} - -interface OracleGeneratorConfiguration extends RouterScreenConfiguration { - judges: readonly FixedTraceProtocolStage[]; -} - -const ROUTER_SCREEN_CONFIGURATIONS: readonly RouterScreenConfiguration[] = Object.freeze([ - { id: 'router-haiku-default', provider: 'anthropic', model: 'claude-haiku-4-5', effort: 'provider_default', pricingProfileId: PRICE.haiku }, - { id: 'router-luna-none', provider: 'openai', model: 'gpt-5.6-luna', effort: 'none', pricingProfileId: PRICE.luna }, - { id: 'router-luna-low', provider: 'openai', model: 'gpt-5.6-luna', effort: 'low', pricingProfileId: PRICE.luna }, - { id: 'router-terra-none', provider: 'openai', model: 'gpt-5.6-terra', effort: 'none', pricingProfileId: PRICE.terra }, - { id: 'router-terra-low', provider: 'openai', model: 'gpt-5.6-terra', effort: 'low', pricingProfileId: PRICE.terra }, - { id: 'router-gemini-low', provider: 'google', model: 'gemini-3.7-flash', effort: 'low', pricingProfileId: PRICE.gemini }, -]); - -const ORACLE_GENERATOR_CONFIGURATIONS: readonly OracleGeneratorConfiguration[] = Object.freeze([ - { id: 'oracle-sonnet-default', provider: 'anthropic', model: 'claude-sonnet-5', effort: 'provider_default', pricingProfileId: PRICE.sonnet, judges: terraAndGeminiJudges }, - { id: 'oracle-sonnet-medium', provider: 'anthropic', model: 'claude-sonnet-5', effort: 'medium', pricingProfileId: PRICE.sonnet, judges: terraAndGeminiJudges }, - { id: 'oracle-terra-low', provider: 'openai', model: 'gpt-5.6-terra', effort: 'low', pricingProfileId: PRICE.terra, judges: sonnetAndGeminiJudges }, - { id: 'oracle-terra-medium', provider: 'openai', model: 'gpt-5.6-terra', effort: 'medium', pricingProfileId: PRICE.terra, judges: sonnetAndGeminiJudges }, - { id: 'oracle-sol-low', provider: 'openai', model: 'gpt-5.6-sol', effort: 'low', pricingProfileId: PRICE.sol, judges: sonnetAndGeminiJudges }, - { id: 'oracle-sol-medium', provider: 'openai', model: 'gpt-5.6-sol', effort: 'medium', pricingProfileId: PRICE.sol, judges: sonnetAndGeminiJudges }, - { id: 'oracle-gemini-medium', provider: 'google', model: 'gemini-3.7-flash', effort: 'medium', pricingProfileId: PRICE.gemini, judges: terraAndSonnetJudges }, -]); - -/** - * The exact conservative approval projection. It is intentionally - * non-dispatchable until a future evaluator-owned trusted manifest binds the - * real 46/36/38 corpus, raw ledger, and direct/hybrid execution contracts. - */ +/** A closed, diagnostic-only projection with no promotion or execution path. */ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = Object.freeze({ version: FIXED_TRACE_EVALUATION_PROTOCOL_VERSION, id: 'addie-6842-6846-staged-v1', trustedManifestId: 'externally-owned-addie-fixed-trace-v120', pricingAsOf: '2026-09-05T12:00:00.000Z', - contingencyBasisPoints: 1_500, + contingencyBasisPoints: 0, phases: Object.freeze([ Object.freeze({ - id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, resultUse: 'smoke_only', + id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, resultUse: 'diagnostic_only', arms: Object.freeze([Object.freeze({ id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', stages: Object.freeze([ @@ -628,69 +517,39 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto })]), }), Object.freeze({ - id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, resultUse: 'component_screening', - arms: Object.freeze(ROUTER_SCREEN_CONFIGURATIONS.map((configuration) => Object.freeze({ - id: configuration.id, architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, - stages: Object.freeze([router(configuration.provider, configuration.model, configuration.effort, configuration.pricingProfileId)]), - }))), + id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, resultUse: 'diagnostic_only', + arms: Object.freeze([Object.freeze({ id: 'router-haiku-default', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]) })]), }), Object.freeze({ - id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, resultUse: 'diagnostic', - arms: Object.freeze(ORACLE_GENERATOR_CONFIGURATIONS.map((configuration) => Object.freeze({ - id: configuration.id, architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, - stages: Object.freeze([ - generation(configuration.provider, configuration.model, configuration.effort, configuration.pricingProfileId), - ...configuration.judges, - ]), - }))), + id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, resultUse: 'diagnostic_only', + arms: Object.freeze([Object.freeze({ id: 'oracle-sonnet-default', architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, stages: Object.freeze([generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet)]) })]), }), Object.freeze({ - id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, resultUse: 'selective', + id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, resultUse: 'diagnostic_only', arms: Object.freeze([ Object.freeze({ id: 'incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ...terraAndGeminiJudges, - ]) }), - Object.freeze({ id: 'openai-luna-low-terra-medium', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), - generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, - ]) }), - Object.freeze({ id: 'hybrid-safe-signal-luna-terra', architecture: 'hybrid_safe_signal_then_llm' as const, admission: 'requires_verified_hybrid_contract' as const, stages: Object.freeze([ - router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), - generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, - ]) }), - Object.freeze({ id: 'openai-luna-low-sol-medium', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), - generation('openai', 'gpt-5.6-sol', 'medium', PRICE.sol), ...sonnetAndGeminiJudges, + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ]) }), Object.freeze({ id: 'gemini-low-medium-pipeline', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ router('google', 'gemini-3.7-flash', 'low', PRICE.gemini), - generation('google', 'gemini-3.7-flash', 'medium', PRICE.gemini), ...terraAndSonnetJudges, - ]) }), - Object.freeze({ id: 'direct-bounded-terra-medium', architecture: 'direct_bounded_production_shaped' as const, admission: 'requires_verified_direct_contract' as const, stages: Object.freeze([ - generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + generation('google', 'gemini-3.7-flash', 'medium', PRICE.gemini), ]) }), ]), }), Object.freeze({ - id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, resultUse: 'selective', + id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, resultUse: 'diagnostic_only', arms: Object.freeze([ Object.freeze({ id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ...terraAndGeminiJudges, - ]) }), - Object.freeze({ id: 'tuning-openai-luna-terra', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ]) }), ]), }), Object.freeze({ - id: 'sealed_final', uniqueCaseCount: 38, repetitions: 3, resultUse: 'promotional', + id: 'sealed_final', uniqueCaseCount: 38, repetitions: 3, resultUse: 'diagnostic_only', arms: Object.freeze([ Object.freeze({ id: 'final-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ...terraAndGeminiJudges, - ]) }), - Object.freeze({ id: 'final-openai-luna-terra', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('openai', 'gpt-5.6-luna', 'low', PRICE.luna), generation('openai', 'gpt-5.6-terra', 'medium', PRICE.terra), ...sonnetAndGeminiJudges, + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ]) }), ]), }), diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts index 5c60dbf352..b14c985c50 100644 --- a/server/src/addie/eval/fixed-trace-experiment-plan.ts +++ b/server/src/addie/eval/fixed-trace-experiment-plan.ts @@ -2,8 +2,6 @@ import { createHash } from 'node:crypto'; import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS, - OPENAI_GPT_5_6_PRICING_VERSION, } from '../model-cost-pricing.js'; import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; import { @@ -16,11 +14,9 @@ import type { AddieTool } from '../types.js'; import { CODE_VERSION } from '../config-version.js'; import { FIXED_TRACE_STAGE_CONTROL_VERSION, - fixedTraceSuiteSha256, + FIXED_TRACE_SUITE, type FixedTraceCase, } from './fixed-trace-suite.js'; -import { fixedTraceToolSchemaSha256 } from './fixed-trace-runner.js'; -import { validateFixedTraceToolLoopFixtures } from './fixed-trace-tool-loop.js'; import type { FixedTraceToolDefinitionProvenance } from './fixed-trace-architecture.js'; /** A versioned, network-free admission contract for fixed-trace experiments. */ @@ -66,24 +62,6 @@ export const FIXED_TRACE_IMMUTABLE_PRICING = Object.freeze([ validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 3, outputUsdPerMillionTokens: 15, source: 'Repository Anthropic standard pricing table, refreshed August 2026.', }), - Object.freeze({ - provider: 'openai', model: 'gpt-5.6-luna', version: OPENAI_GPT_5_6_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-luna'].inputUsd, - outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-luna'].outputUsd, - source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', - }), - Object.freeze({ - provider: 'openai', model: 'gpt-5.6-terra', version: OPENAI_GPT_5_6_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-terra'].inputUsd, - outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-terra'].outputUsd, - source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', - }), - Object.freeze({ - provider: 'openai', model: 'gpt-5.6-sol', version: OPENAI_GPT_5_6_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-sol'].inputUsd, - outputUsdPerMillionTokens: OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS['gpt-5.6-sol'].outputUsd, - source: 'Repository immutable OpenAI standard pricing pin, reviewed 2026-09-05.', - }), Object.freeze({ provider: 'google', model: 'gemini-3.7-flash', version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, validBefore: '2027-01-01T00:00:00.000Z', inputUsdPerMillionTokens: 0.75, outputUsdPerMillionTokens: 3.75, @@ -262,6 +240,11 @@ export interface FixedTraceRawLedgerEntry { reasoningEffort: ModelReasoningEffort; samplingMode: 'temperature_zero' | 'provider_no_sampling_control' | null; cacheMode: 'disabled' | null; + /** Offline validation admits only the explicit non-dispatch terminal state. */ + status: 'not_dispatched'; + finishReason: null; + usage: null; + estimatedCostUsd: null; } export interface FixedTraceRawAuditableLedger { @@ -300,6 +283,70 @@ export interface FixedTraceDryRunEstimate { expectedSpendUsd: null; } +export interface FixedTraceOfflinePlanValidation { + diagnosticOnly: true; + comparisonEligible: false; + dispatchable: false; + trustedLock: false; + planFingerprint: string; +} + +/** + * Copies only JSON data from a plain object. Reflection happens before any + * value read, so accessors are never invoked. `structuredClone` rejects a + * Proxy, which closes the remaining caller-controlled object membrane. + */ +function snapshotJson(value: unknown, label: string): unknown { + const copy = (candidate: unknown, path: string): unknown => { + if (candidate === null || typeof candidate === 'string' || typeof candidate === 'boolean') return candidate; + if (typeof candidate === 'number') { + if (!Number.isFinite(candidate)) throw new Error(`${path} contains a non-finite number`); + return candidate; + } + if (typeof candidate !== 'object') throw new Error(`${path} is not JSON data`); + if (Array.isArray(candidate)) { + const descriptors = Object.getOwnPropertyDescriptors(candidate); + if (Object.getPrototypeOf(candidate) !== Array.prototype || Object.getOwnPropertySymbols(candidate).length > 0) { + throw new Error(`${path} must be a plain array without symbols`); + } + for (const [key, descriptor] of Object.entries(descriptors)) { + if (key !== 'length' && (!('value' in descriptor) || !descriptor.enumerable)) { + throw new Error(`${path} contains an accessor or hidden property`); + } + } + return candidate.map((item, index) => copy(item, `${path}[${index}]`)); + } + if (Object.getPrototypeOf(candidate) !== Object.prototype || Object.getOwnPropertySymbols(candidate).length > 0) { + throw new Error(`${path} must be a plain object without symbols`); + } + const descriptors = Object.getOwnPropertyDescriptors(candidate); + const output: Record = {}; + for (const [key, descriptor] of Object.entries(descriptors)) { + if (!('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}.${key} must be an own enumerable data property`); + } + output[key] = copy(descriptor.value, `${path}.${key}`); + } + return output; + }; + // Do this after descriptor validation: structuredClone otherwise invokes a + // getter. It reliably rejects Proxy values that can impersonate descriptors. + try { + structuredClone(value); + } catch { + throw new Error(`${label} must not contain a Proxy or non-cloneable value`); + } + return deepFreeze(copy(value, label)); +} + +function assertExactKeys(value: object, keys: readonly string[], label: string): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new Error(`${label} has unknown, missing, or inherited fields`); + } +} + function canonicalJson(value: unknown): string { if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); if (typeof value === 'number') { @@ -355,6 +402,13 @@ function validateStage( traceIds: readonly string[], pricingAsOf: string, ): FixedTraceImmutablePricingProfile { + assertExactKeys(stage, [ + 'provider', 'model', 'reasoningEffort', 'pricingVersion', 'maxOutputTokens', + 'timeoutMs', 'maxIterations', 'transportRetries', 'samplingMode', 'temperature', + 'cacheMode', 'requestBounds', + ], label); + if (!['anthropic', 'openai', 'google'].includes(stage.provider)) throw new Error(`${label}.provider is unknown`); + if (!['provider_default', 'none', 'low', 'medium', 'high'].includes(stage.reasoningEffort)) throw new Error(`${label}.reasoningEffort is unknown`); if (!stage.model.trim()) throw new Error(`${label}.model is required`); requirePositiveInteger(stage.maxOutputTokens, `${label}.maxOutputTokens`); requirePositiveInteger(stage.timeoutMs, `${label}.timeoutMs`); @@ -369,6 +423,8 @@ function validateStage( const pricing = pricingFor(stage, pricingAsOf); const bounds = stage.requestBounds?.inputBytesByTrace; if (!bounds || typeof bounds !== 'object') throw new Error(`${label}.requestBounds are required`); + assertExactKeys(stage.requestBounds, ['inputBytesByTrace'], `${label}.requestBounds`); + assertExactKeys(bounds, traceIds, `${label}.requestBounds.inputBytesByTrace`); for (const traceId of traceIds) { const values = bounds[traceId]; if (!Array.isArray(values) || values.length !== stage.maxIterations) { @@ -383,6 +439,14 @@ function validateStage( } function validateArm(plan: FixedTraceExperimentPlan, arm: FixedTraceExperimentArm): void { + assertExactKeys(arm, ['id', 'architecture', 'screeningStage', 'repetitionIndex', ...( + arm.router ? ['router'] : []), ...(arm.generation ? ['generation'] : []), ...(arm.judges ? ['judges'] : [])], `arm ${arm.id}`); + if (!['two_stage_llm_router', 'direct_generation', 'hybrid_generation', 'oracle_route_diagnostic'].includes(arm.architecture)) { + throw new Error(`${arm.id}.architecture is unknown`); + } + if (!['router_only_screen', 'oracle_route_generator_diagnostic', 'deployable_finalist'].includes(arm.screeningStage)) { + throw new Error(`${arm.id}.screeningStage is unknown`); + } if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid experiment arm ID: ${arm.id}`); requirePositiveInteger(arm.repetitionIndex, `${arm.id}.repetitionIndex`); const traces = selectedTraceIds(plan); @@ -410,6 +474,11 @@ function validateArm(plan: FixedTraceExperimentPlan, arm: FixedTraceExperimentAr const candidateProviders = new Set([arm.router.provider, arm.generation.provider]); const judgeProviders = new Set(); for (const [index, judge] of arm.judges.entries()) { + assertExactKeys(judge, [ + 'provider', 'model', 'reasoningEffort', 'pricingVersion', 'maxOutputTokens', + 'timeoutMs', 'maxIterations', 'transportRetries', 'samplingMode', 'temperature', + 'cacheMode', 'requestBounds', 'blinded', + ], `${arm.id}.judges.${index}`); if (judge.blinded !== true) throw new Error(`${arm.id}.judges.${index} must be blinded`); if (candidateProviders.has(judge.provider)) throw new Error(`${arm.id}.judges.${index} is not provider-independent`); judgeProviders.add(judge.provider); @@ -418,66 +487,155 @@ function validateArm(plan: FixedTraceExperimentPlan, arm: FixedTraceExperimentAr if (judgeProviders.size < 2) throw new Error(`${arm.id} requires two provider-independent judges`); } -/** Validates without loading trace fixtures, prompts, credentials, or providers. */ -function resolveTrustedManifest( +function assertPlanShape(plan: FixedTraceExperimentPlan): void { + assertExactKeys(plan, [ + 'version', 'id', 'trustedManifestId', 'sourceId', 'sourceRevision', 'pricingAsOf', + 'sourceBundleSha256', 'gitCommit', 'gitDirty', 'addieCodeVersion', 'stageControlVersion', + 'traceSuiteSha256', 'promptConfigVersion', 'toolSchemaSha256', 'toolDefinitionProvenance', + 'providerDegradationInjectionEnabled', 'partition', 'ordering', 'budgets', 'arms', + ], 'experiment plan'); + assertExactKeys(plan.partition, [ + 'manifestVersion', 'manifestSha256', 'selected', ...(plan.partition.finalizationGate ? ['finalizationGate'] : []), + ], 'experiment plan.partition'); + if (plan.partition.finalizationGate) { + assertExactKeys(plan.partition.finalizationGate, ['version', 'recordId'], 'experiment plan.partition.finalizationGate'); + } + assertExactKeys(plan.ordering, ['seed'], 'experiment plan.ordering'); + assertExactKeys(plan.budgets, ['candidateCeilingUsd', 'judgeCeilingUsd'], 'experiment plan.budgets'); +} + +function assertFixedTraceExperimentPlanStructure( plan: FixedTraceExperimentPlan, - resolver: FixedTraceTrustedManifestResolver, -): FixedTraceTrustedManifest { - const manifest = resolver(plan.trustedManifestId); - if (!manifest) throw new Error(`Trusted fixed-trace manifest is unavailable: ${plan.trustedManifestId}`); - requireHash(manifest.sourceBundleSha256, 'trusted manifest sourceBundleSha256'); - requireHash(manifest.promptConfigVersion, 'trusted manifest promptConfigVersion'); - if (manifest.rawLedgerVersion !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Trusted manifest requires an unsupported raw ledger'); - if ( - !/^[a-f0-9]{7,64}$/.test(manifest.gitCommit) - || typeof manifest.gitDirty !== 'boolean' - || !manifest.addieCodeVersion.trim() - || manifest.stageControlVersion !== FIXED_TRACE_STAGE_CONTROL_VERSION - || typeof manifest.providerDegradationInjectionEnabled !== 'boolean' - ) throw new Error('Trusted manifest run provenance is incomplete'); - const suite = manifest.suites[plan.partition.selected]; - if (!suite) throw new Error(`Trusted manifest lacks ${plan.partition.selected} suite inputs`); - requireHash(suite.traceSuiteSha256, 'trusted manifest traceSuiteSha256'); - requireHash(suite.toolSchemaSha256, 'trusted manifest toolSchemaSha256'); - if (!Array.isArray(suite.traceSuite)) throw new Error('Trusted manifest suite does not exactly bind the selected partition'); - const selectedIds = selectedTraceIds(plan); - const suiteIds = suite.traceSuite.map((trace) => trace.id); - if ( - suite.traceSuite.length !== selectedIds.length - || suiteIds.some((id) => !id.trim()) - || new Set(suiteIds).size !== suiteIds.length - || suiteIds.some((id) => !selectedIds.includes(id)) - || new Set(selectedIds).size !== new Set(suiteIds).size - ) throw new Error('Trusted manifest suite does not exactly bind the selected partition'); - if ( - fixedTraceSuiteSha256(suite.traceSuite) !== suite.traceSuiteSha256 - || fixedTraceToolSchemaSha256(suite.traceSuite, suite.toolDefinitions) !== suite.toolSchemaSha256 - ) throw new Error('Trusted manifest suite or tool schema hash is forged'); - // This is the same fixture registration/AJV primitive the foundation uses - // before routing. Direct remains inadmissible in this planner, so no fake - // fixture-derived direct universe is accepted here. - for (const trace of suite.traceSuite) { - const definitions = suite.toolDefinitions.filter((definition) => - trace.toolFixtures.some((fixture: FixedTraceCase['toolFixtures'][number]) => fixture.name === definition.name)); - validateFixedTraceToolLoopFixtures(trace, definitions); + holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, +): void { + assertPlanShape(plan); + assertFixedTracePartitionManifest(); + if (plan.version !== FIXED_TRACE_EXPERIMENT_PLAN_VERSION) throw new Error('Unsupported fixed-trace experiment plan version'); + if (!plan.id.trim()) throw new Error('Experiment plan ID is required'); + if (!plan.trustedManifestId.trim() || !plan.sourceId.trim() || !plan.sourceRevision.trim()) throw new Error('Experiment plan requires a trusted source identity'); + requireHash(plan.sourceBundleSha256, 'sourceBundleSha256'); + if (!/^[a-f0-9]{7,64}$/.test(plan.gitCommit) || typeof plan.gitDirty !== 'boolean' || !plan.addieCodeVersion.trim() || plan.stageControlVersion !== FIXED_TRACE_STAGE_CONTROL_VERSION || typeof plan.providerDegradationInjectionEnabled !== 'boolean') { + throw new Error('Experiment plan run provenance is incomplete'); } - if ( - manifest.id !== plan.trustedManifestId - || manifest.sourceId !== plan.sourceId - || manifest.sourceRevision !== plan.sourceRevision - || manifest.sourceBundleSha256 !== plan.sourceBundleSha256 - || suite.traceSuiteSha256 !== plan.traceSuiteSha256 - || manifest.promptConfigVersion !== plan.promptConfigVersion - || suite.toolSchemaSha256 !== plan.toolSchemaSha256 - || manifest.gitCommit !== plan.gitCommit - || manifest.gitDirty !== plan.gitDirty - || manifest.addieCodeVersion !== plan.addieCodeVersion - || suite.toolDefinitionProvenance !== plan.toolDefinitionProvenance - || manifest.stageControlVersion !== plan.stageControlVersion - || manifest.providerDegradationInjectionEnabled !== plan.providerDegradationInjectionEnabled - || manifest.partitionManifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256 - ) throw new Error('Experiment plan does not match its trusted manifest'); - return manifest; + if (plan.addieCodeVersion !== CODE_VERSION) throw new Error('Experiment plan Addie code version does not match this runner'); + requireHash(plan.traceSuiteSha256, 'traceSuiteSha256'); + requireHash(plan.promptConfigVersion, 'promptConfigVersion'); + requireHash(plan.toolSchemaSha256, 'toolSchemaSha256'); + if (plan.partition.manifestVersion !== FIXED_TRACE_PARTITION_MANIFEST_VERSION || plan.partition.manifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { + throw new Error('Experiment plan uses an uncommitted fixed-trace partition manifest'); + } + if (plan.partition.selected === 'holdout') { + assertHoldoutFinalization(plan, holdoutFinalizationResolver); + } else if (plan.partition.selected !== 'development' || plan.partition.finalizationGate) { + throw new Error('Development execution must not carry a holdout finalization gate'); + } + if (!plan.ordering.seed.trim()) throw new Error('Experiment ordering seed is required'); + if (!Number.isFinite(plan.budgets.candidateCeilingUsd) || plan.budgets.candidateCeilingUsd <= 0) throw new Error('candidateCeilingUsd must be positive'); + if (!Number.isFinite(plan.budgets.judgeCeilingUsd) || plan.budgets.judgeCeilingUsd <= 0) throw new Error('judgeCeilingUsd must be positive'); + if (!Array.isArray(plan.arms) || plan.arms.length === 0) throw new Error('Experiment plan requires at least one arm'); + const ids = new Set(); + for (const arm of plan.arms) { + if (ids.has(arm.id)) throw new Error(`Duplicate experiment arm ID: ${arm.id}`); + ids.add(arm.id); + validateArm(plan, arm); + } +} + +/** Validate an untrusted plan without credentials, providers, outputs, or a resolver. */ +export function validateFixedTraceExperimentPlanOffline(plan: FixedTraceExperimentPlan): FixedTraceOfflinePlanValidation { + const snapshot = snapshotJson(plan, 'experiment plan') as FixedTraceExperimentPlan; + // A submitted plan can describe only priced, already reviewed stages. Terra + // and Sol have no reviewed repository price, so their descriptors cannot + // enter an estimate or a budget reservation. + assertFixedTraceExperimentPlanStructure(snapshot); + return Object.freeze({ + diagnosticOnly: true, + comparisonEligible: false, + dispatchable: false, + trustedLock: false, + planFingerprint: sha256(snapshot), + }); +} + +/** + * Validates an offline ledger in its exact planned order. Nothing in this + * lane can have been dispatched, returned, priced, or promoted; accepting a + * partial or provider-shaped record would let caller data masquerade as + * evidence. + */ +export function validateFixedTraceRawAuditableLedgerOffline( + plan: FixedTraceExperimentPlan, + ledger: FixedTraceRawAuditableLedger, +): void { + const safePlan = snapshotJson(plan, 'experiment plan') as FixedTraceExperimentPlan; + const safeLedger = snapshotJson(ledger, 'raw ledger') as FixedTraceRawAuditableLedger; + assertFixedTraceExperimentPlanStructure(safePlan); + assertExactKeys(safeLedger, ['version', 'trustedManifestSha256', 'planFingerprint', 'budgetIdentitySha256', 'entries'], 'raw ledger'); + if (safeLedger.version !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Unsupported raw fixed-trace ledger version'); + if (!Array.isArray(safeLedger.entries)) throw new Error('Raw ledger entries must be an array'); + const traceById = new Map(FIXED_TRACE_SUITE.map((trace) => [trace.id, trace])); + const expected = safePlan.arms.flatMap((arm) => selectedTraceIds(safePlan).flatMap((traceId) => { + const stages: Array<[FixedTraceRawLedgerEntry['stage'], FixedTracePlannedStage]> = []; + if (arm.router) stages.push(['router', arm.router]); + if (arm.generation) stages.push(['generation', arm.generation]); + for (const judge of arm.judges ?? []) stages.push(['judge', judge]); + return stages.map(([stage, configuredStage]) => ({ arm, traceId, stage, configuredStage })); + })); + if (safeLedger.entries.length !== expected.length) throw new Error('Raw ledger lacks complete planned-stage coverage'); + for (const [index, entry] of safeLedger.entries.entries()) { + assertExactKeys(entry, [ + 'sequence', 'armId', 'repetitionIndex', 'traceId', 'stage', 'dispatched', + 'requestedProvider', 'requestedModel', 'returnedProvider', 'returnedModel', + 'promptSha256', 'providerRequestSha256', 'responseSha256', 'rawRequestArtifact', + 'rawResponseArtifact', 'exactToolNames', 'caseControlSha256', 'executionEnvelopeSha256', + 'directAdmissionSha256', 'maxOutputTokens', 'timeoutMs', 'maxIterations', + 'transportRetries', 'reasoningEffort', 'samplingMode', 'cacheMode', 'status', + 'finishReason', 'usage', 'estimatedCostUsd', + ], `raw ledger entry ${index + 1}`); + const want = expected[index]!; + if (entry.sequence !== index + 1 || entry.armId !== want.arm.id || entry.repetitionIndex !== want.arm.repetitionIndex || entry.traceId !== want.traceId || entry.stage !== want.stage) { + throw new Error('Raw ledger sequence does not exactly match the planned stages'); + } + const trace = traceById.get(entry.traceId); + const exactToolNames: readonly string[] = trace ? trace.toolFixtures.map((fixture) => fixture.name) : []; + if (!trace || !Array.isArray(entry.exactToolNames) || entry.exactToolNames.length !== exactToolNames.length || entry.exactToolNames.some((name: string, toolIndex: number) => name !== exactToolNames[toolIndex])) { + throw new Error('Raw ledger tool names do not exactly match the trace fixtures'); + } + if ( + entry.dispatched !== false || entry.status !== 'not_dispatched' || entry.finishReason !== null + || entry.usage !== null || entry.estimatedCostUsd !== null || entry.returnedProvider !== null + || entry.returnedModel !== null || entry.providerRequestSha256 !== null || entry.responseSha256 !== null + || entry.rawRequestArtifact !== null || entry.rawResponseArtifact !== null + ) throw new Error('Offline raw ledger contains dispatch, response, usage, or cost evidence'); + if (entry.requestedProvider !== want.configuredStage.provider || entry.requestedModel !== want.configuredStage.model) { + throw new Error('Raw ledger requested provider/model does not match its planned stage'); + } + for (const [label, value] of Object.entries({ + promptSha256: entry.promptSha256, + caseControlSha256: entry.caseControlSha256, + executionEnvelopeSha256: entry.executionEnvelopeSha256, + directAdmissionSha256: entry.directAdmissionSha256, + })) requireHash(value, `raw ledger ${label}`); + if ( + entry.maxOutputTokens !== want.configuredStage.maxOutputTokens + || entry.timeoutMs !== want.configuredStage.timeoutMs + || entry.maxIterations !== want.configuredStage.maxIterations + || entry.transportRetries !== 0 || entry.reasoningEffort !== want.configuredStage.reasoningEffort + || entry.samplingMode !== want.configuredStage.samplingMode || entry.cacheMode !== 'disabled' + ) throw new Error('Raw ledger entry does not match its planned stage controls'); + } +} + +/** + * There is deliberately no resolver in this change that can turn caller JSON + * into evaluator authority. A future reviewed evaluator must authenticate a + * manifest outside this process before exposing an execution binding. + */ +function resolveTrustedManifest( + _plan: FixedTraceExperimentPlan, + _resolver: FixedTraceTrustedManifestResolver, +): FixedTraceTrustedManifest { + throw new Error('Trusted fixed-trace manifest is locked pending evaluator-owned authentication'); } /** Builds an immutable input binding for exactly one planned arm, never a dispatcher. */ @@ -546,47 +704,22 @@ export function assertFixedTraceExperimentPlan( resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, ): void { - assertFixedTracePartitionManifest(); - if (plan.version !== FIXED_TRACE_EXPERIMENT_PLAN_VERSION) throw new Error('Unsupported fixed-trace experiment plan version'); - if (!plan.id.trim()) throw new Error('Experiment plan ID is required'); - if (!plan.trustedManifestId.trim() || !plan.sourceId.trim() || !plan.sourceRevision.trim()) throw new Error('Experiment plan requires a trusted source identity'); - requireHash(plan.sourceBundleSha256, 'sourceBundleSha256'); - if (!/^[a-f0-9]{7,64}$/.test(plan.gitCommit) || typeof plan.gitDirty !== 'boolean' || !plan.addieCodeVersion.trim() || plan.stageControlVersion !== FIXED_TRACE_STAGE_CONTROL_VERSION || typeof plan.providerDegradationInjectionEnabled !== 'boolean') { - throw new Error('Experiment plan run provenance is incomplete'); - } - if (plan.addieCodeVersion !== CODE_VERSION) throw new Error('Experiment plan Addie code version does not match this runner'); - requireHash(plan.traceSuiteSha256, 'traceSuiteSha256'); - requireHash(plan.promptConfigVersion, 'promptConfigVersion'); - requireHash(plan.toolSchemaSha256, 'toolSchemaSha256'); - if (plan.partition.manifestVersion !== FIXED_TRACE_PARTITION_MANIFEST_VERSION || plan.partition.manifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { - throw new Error('Experiment plan uses an uncommitted fixed-trace partition manifest'); - } - if (plan.partition.selected === 'holdout') { - assertHoldoutFinalization(plan, holdoutFinalizationResolver); - } else if (plan.partition.finalizationGate) { - throw new Error('Development execution must not carry a holdout finalization gate'); - } - if (!plan.ordering.seed.trim()) throw new Error('Experiment ordering seed is required'); - if (!Number.isFinite(plan.budgets.candidateCeilingUsd) || plan.budgets.candidateCeilingUsd <= 0) throw new Error('candidateCeilingUsd must be positive'); - if (!Number.isFinite(plan.budgets.judgeCeilingUsd) || plan.budgets.judgeCeilingUsd <= 0) throw new Error('judgeCeilingUsd must be positive'); - if (!Array.isArray(plan.arms) || plan.arms.length === 0) throw new Error('Experiment plan requires at least one arm'); - const ids = new Set(); - for (const arm of plan.arms) { - if (ids.has(arm.id)) throw new Error(`Duplicate experiment arm ID: ${arm.id}`); - ids.add(arm.id); - validateArm(plan, arm); - } - resolveTrustedManifest(plan, resolver); + const snapshot = snapshotJson(plan, 'experiment plan') as FixedTraceExperimentPlan; + assertFixedTraceExperimentPlanStructure(snapshot, holdoutFinalizationResolver); + resolveTrustedManifest(snapshot, resolver); } export function fixedTraceExperimentPlanFingerprint(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): string { - assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); - return sha256(plan); + void resolver; + void holdoutFinalizationResolver; + return validateFixedTraceExperimentPlanOffline(plan).planFingerprint; } /** Deterministic permutation based on a recorded seed, never provider input order. */ export function fixedTraceExperimentExecutionOrder(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): readonly string[] { - assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + void resolver; + void holdoutFinalizationResolver; + validateFixedTraceExperimentPlanOffline(plan); return Object.freeze([...plan.arms] .sort((left, right) => sha256({ seed: plan.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) .localeCompare(sha256({ seed: plan.ordering.seed, arm: right.id, repetition: right.repetitionIndex })) || left.id.localeCompare(right.id)) @@ -615,7 +748,9 @@ function reservation( * provider tokenization nor observed tool-loop length may be assumed. */ export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): FixedTraceDryRunEstimate { - assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + void resolver; + void holdoutFinalizationResolver; + validateFixedTraceExperimentPlanOffline(plan); const candidate: FixedTraceStageReservation[] = []; const judges: FixedTraceStageReservation[] = []; const traceIds = selectedTraceIds(plan); diff --git a/server/src/addie/model-cost-pricing.ts b/server/src/addie/model-cost-pricing.ts index 259aeaaf7e..5397fdeca1 100644 --- a/server/src/addie/model-cost-pricing.ts +++ b/server/src/addie/model-cost-pricing.ts @@ -20,18 +20,6 @@ import type { ModelProviderId, ModelUsage } from './model-providers/model-provid export const GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION = 'google-gemini-3.7-flash-through-2026-12-31' as const; -/** - * Immutable OpenAI standard rates reviewed for the fixed-trace planning - * contract on 2026-09-05. These are deliberately model-specific; callers - * must not infer a rate for a new model or a returned revision suffix. - */ -export const OPENAI_GPT_5_6_PRICING_VERSION = 'openai-gpt-5.6-standard-2026-09-05' as const; -export const OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS = Object.freeze({ - 'gpt-5.6-luna': Object.freeze({ inputUsd: 0.20, outputUsd: 1.20 }), - 'gpt-5.6-terra': Object.freeze({ inputUsd: 2, outputUsd: 12 }), - 'gpt-5.6-sol': Object.freeze({ inputUsd: 4, outputUsd: 20 }), -} as const); - export interface ModelCostPricing { provider: ModelProviderId; model: string; @@ -112,22 +100,5 @@ export function resolveModelCostPricing( }, }; } - if (provider === 'openai') { - const rate = OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS[ - model as keyof typeof OPENAI_GPT_5_6_PRICING_PER_MILLION_TOKENS - ]; - if (!rate) return null; - return { - provider: 'openai', - model, - version: OPENAI_GPT_5_6_PRICING_VERSION, - // The plan contract is intentionally date-pinned. A later plan must - // add a reviewed profile instead of silently reusing this rate. - validBefore: new Date('2026-09-06T00:00:00.000Z'), - estimateCostMicros: (usage) => Math.ceil( - usage.inputTokens * rate.inputUsd + usage.outputTokens * rate.outputUsd, - ), - }; - } return null; } diff --git a/server/src/addie/model-providers/openai-responses-provider.ts b/server/src/addie/model-providers/openai-responses-provider.ts index 612abe6eac..c52e1ab1e3 100644 --- a/server/src/addie/model-providers/openai-responses-provider.ts +++ b/server/src/addie/model-providers/openai-responses-provider.ts @@ -23,22 +23,6 @@ import { assertPlainJson, validateModelCapabilities } from './capabilities.js'; import { validateNormalizedModelResponse } from './events.js'; export const OPENAI_ROUTER_MODEL = 'gpt-5.6-luna'; -/** - * Exact OpenAI model identifiers reviewed for offline fixed-trace planning. - * Keep this list explicit: provider-returned revision suffixes are provenance, - * never aliases accepted at the request boundary. - */ -export const OPENAI_FIXED_TRACE_MODELS = Object.freeze([ - 'gpt-5.6-luna', - 'gpt-5.6-terra', - 'gpt-5.6-sol', -] as const); - -export type OpenAIFixedTraceModel = (typeof OPENAI_FIXED_TRACE_MODELS)[number]; - -export function isSupportedOpenAIResponsesModel(model: string): model is OpenAIFixedTraceModel { - return (OPENAI_FIXED_TRACE_MODELS as readonly string[]).includes(model); -} export interface OpenAIResponsesTransport { responses: { @@ -155,9 +139,7 @@ function toOpenAIInput(messages: readonly ModelMessage[]): ResponseInputItem[] { function toOpenAIRequest(request: ModelRequest): ResponseCreateParamsNonStreaming { validateModelCapabilities('openai', OPENAI_RESPONSES_CAPABILITIES, request); - if (!isSupportedOpenAIResponsesModel(request.model)) { - // Kept for compatibility with existing caller diagnostics; the supported - // set now also includes explicitly planned generation/control models. + if (request.model !== OPENAI_ROUTER_MODEL) { throw new Error(`Unsupported OpenAI router model: ${request.model}`); } if (request.system.some((block) => block.cacheHint !== undefined)) { diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index 115cf9229f..11f544366f 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -22,14 +22,13 @@ * provider or reads credentials. A live replay must be added as a separately * reviewed consumer of the versioned plan contract. * - * Example: - * npm run eval:addie-fixed-traces -- \ - * --experiment-plan=.context/evals/plan.json \ - * --trusted-manifest=.context/evals/trusted-manifest.json + * `--experiment-plan` and `--trusted-manifest` are accepted only with + * `--validate-only`; neither can make a caller-built manifest trusted. */ import { createHash, randomUUID } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { resolve } from 'node:path'; +import { readFileSync } from 'node:fs'; import { ModelConfig } from '../../src/config/models.js'; import { CODE_VERSION, computeRouterRulesHash } from '../../src/addie/config-version.js'; import { @@ -46,6 +45,7 @@ import { } from '../../src/addie/eval/fixed-trace-diagnostic-run.js'; import { MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS } from '../../src/addie/eval/fixed-trace-tool-loop.js'; import { parseFixedTraceDiagnosticCliArguments } from '../../src/addie/eval/fixed-trace-diagnostic-cli.js'; +import { validateFixedTraceExperimentPlanOffline } from '../../src/addie/eval/fixed-trace-experiment-plan.js'; import { reserveFixedTraceDiagnosticOutput } from '../../src/addie/eval/fixed-trace-diagnostic-output.js'; import { canonicalFixedTraceToolDefinitions } from '../../src/addie/eval/fixed-trace-tools.js'; import { @@ -125,7 +125,7 @@ const PRICING = { const cliArguments = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); function argument(name: string): string | undefined { - return cliArguments[{ providers: 'providers', 'architecture-arm': 'architectureArm', suite: 'suite', 'soft-max-usd': 'softMaxUsd', output: 'output' }[name] as keyof typeof cliArguments] as string | undefined; + return cliArguments[{ providers: 'providers', 'architecture-arm': 'architectureArm', suite: 'suite', 'soft-max-usd': 'softMaxUsd', output: 'output', 'experiment-plan': 'experimentPlan', 'trusted-manifest': 'trustedManifest' }[name] as keyof typeof cliArguments] as string | undefined; } function sha256(value: string): string { @@ -298,9 +298,18 @@ if (!Number.isFinite(softMaxUsd) || softMaxUsd <= 0) { throw new Error('--soft-max-usd is required and must be positive'); } const outputArgument = argument('output'); -if (!outputArgument?.trim()) throw new Error('--output is required'); -const outputPath = resolve(outputArgument); if (cliArguments.validateOnly) { + const experimentPlanPath = argument('experiment-plan'); + const trustedManifestPath = argument('trusted-manifest'); + if ((experimentPlanPath === undefined) !== (trustedManifestPath === undefined)) { + throw new Error('--experiment-plan and --trusted-manifest must be supplied together'); + } + const offlinePlan = experimentPlanPath + ? validateFixedTraceExperimentPlanOffline(JSON.parse(readFileSync(resolve(experimentPlanPath), 'utf8'))) + : undefined; + // Deliberately parse for malformed-file feedback but never deserialize this + // into a resolver or treat it as authority. Authentication is absent here. + if (trustedManifestPath) JSON.parse(readFileSync(resolve(trustedManifestPath), 'utf8')); console.log(JSON.stringify({ diagnosticOnly: true, judgeDispatch: 'blocked_pending_trusted_evaluator_owned_coordinator', @@ -309,11 +318,15 @@ if (cliArguments.validateOnly) { architectureArm, suite: suiteName, softMaxUsd, - outputPath, + outputPath: outputArgument ? resolve(outputArgument) : undefined, + trustedManifestPath: trustedManifestPath ? resolve(trustedManifestPath) : undefined, + offlinePlan, }, })); process.exit(0); } +if (!outputArgument?.trim()) throw new Error('--output is required'); +const outputPath = resolve(outputArgument); throw new Error('Live fixed-trace replay is disabled pending an evaluator-owned execution-contract review'); // This exclusive create happens before source inspection, credentials, // provider construction, or dispatch. Never unlink it: an empty file is the diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts index b98706c109..87d553d706 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts @@ -7,7 +7,7 @@ import { parseFixedTraceDiagnosticCliArguments } from '../../../src/addie/eval/f describe('fixed-trace diagnostic CLI parser', () => { it('accepts only bounded dry-run forms', () => { expect(parseFixedTraceDiagnosticCliArguments(['--validate-only', '--providers=openai'])) - .toEqual({ validateOnly: true, providers: 'openai', architectureArm: undefined, suite: undefined, softMaxUsd: undefined, output: undefined }); + .toEqual({ validateOnly: true, providers: 'openai', architectureArm: undefined, suite: undefined, softMaxUsd: undefined, output: undefined, experimentPlan: undefined, trustedManifest: undefined }); expect(parseFixedTraceDiagnosticCliArguments(['--validate-only=true']).validateOnly).toBe(true); }); @@ -50,8 +50,8 @@ describe('fixed-trace diagnostic CLI parser', () => { it.each([ ['--soft-max-usd=0', '--output=/tmp/out.json'], - ['--soft-max-usd=1'], - ])('rejects incomplete dry run configuration', (...args) => { + ['--experiment-plan=/tmp/no-plan.json'], + ])('rejects malformed dry run configuration', (...args) => { expect(() => execFileSync('npx', [ 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', ...args, ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index 4199ecab60..e9a434a5a4 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -1,135 +1,28 @@ import { describe, expect, it } from 'vitest'; -import { - FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, - assertFixedTraceEvaluationProtocol, - assertFixedTraceEvaluationProtocolTrusted, - estimateFixedTraceEvaluationProtocol, - fixedTraceEvaluationProtocolFingerprint, - fixedTraceEvaluationProtocolRunnerBinding, - type FixedTraceEvaluationProtocol, - type FixedTraceProtocolPhaseId, -} from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; -import { - FIXED_TRACE_SUITE, - fixedTraceSuiteSha256, - type FixedTraceCase, -} from '../../../src/addie/eval/fixed-trace-suite.js'; - -function protocol(): FixedTraceEvaluationProtocol { - return structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); -} - -function evaluatorOwnedSuite(phaseId: FixedTraceProtocolPhaseId, count: number): FixedTraceCase[] { - const template = FIXED_TRACE_SUITE[0]!; - return Array.from({ length: count }, (_, index) => ({ - ...structuredClone(template), - id: `externally-owned-${phaseId}-${index + 1}`, - })); -} +import { FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; describe('fixed-trace evaluation protocol projection', () => { - it('is an exact, non-dispatchable ceiling with staged case and call counts', () => { - const estimate = estimateFixedTraceEvaluationProtocol(protocol()); - expect(estimate.dispatchable).toBe(false); - expect(estimate.expectedSpendUsd).toBeNull(); - expect(estimate.phases.map((phase) => [phase.phaseId, phase.uniqueCaseCount, phase.repetitions, phase.candidateCalls, phase.judgeCalls])).toEqual([ - ['bounded_smoke', 8, 1, 104, 0], - ['router_screen', 46, 3, 828, 0], - ['oracle_generator_ceiling', 46, 2, 7_728, 1_288], - ['deployable_architecture', 46, 3, 10_626, 1_656], - ['controlled_tuning', 36, 3, 2_808, 432], - ['sealed_final', 38, 3, 2_964, 456], - ]); - expect(estimate.stages.every((stage) => stage.cacheMode === 'disabled')).toBe(true); - expect(estimate.stages.every((stage) => stage.inputTokenCeiling === stage.requests * ( - stage.role === 'router' ? 4_096 : stage.role === 'generation' ? 16_384 : 8_192 - ))).toBe(true); - expect(estimate.screening.totalCeilingUsd).toBeGreaterThan(0); - expect(estimate.finalConfirmation.totalCeilingUsd).toBeGreaterThan(0); - expect(estimate.totalCeilingUsd).toBe( - estimate.candidateCeilingUsd + estimate.judgeCeilingUsd + estimate.contingencyUsd, - ); - expect(estimate.approvalCeilingUsd).toBe(1_491); - }); - - it('keeps model, effort, output, cache, and timeout controls in the fingerprint', () => { - const baseline = protocol(); - const changed = protocol(); - changed.phases[1].arms[1].stages[0].reasoningEffort = 'low'; - expect(fixedTraceEvaluationProtocolFingerprint(changed)).not.toBe( - fixedTraceEvaluationProtocolFingerprint(baseline), - ); - changed.phases[1].arms[1].stages[0].maxOutputTokensPerInvocation++; - expect(estimateFixedTraceEvaluationProtocol(changed).totalCeilingUsd).toBeGreaterThan( - estimateFixedTraceEvaluationProtocol(baseline).totalCeilingUsd, - ); + it('is ordered, diagnostic-only, non-dispatchable, and non-promotional', () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + assertFixedTraceEvaluationProtocol(protocol); + expect(protocol.phases.map((phase) => phase.id)).toEqual(['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning', 'sealed_final']); + expect(protocol.phases.every((phase) => phase.resultUse === 'diagnostic_only')).toBe(true); + expect(estimateFixedTraceEvaluationProtocol(protocol)).toMatchObject({ dispatchable: false, expectedSpendUsd: null }); }); - - it('fails closed for unavailable pricing, self-judging, and mixed contracts', () => { - const unknownPricing = protocol(); - unknownPricing.phases[1].arms[0].stages[0].pricingProfileId = 'unknown'; - expect(() => estimateFixedTraceEvaluationProtocol(unknownPricing)).toThrow('Unavailable immutable pricing profile'); - - const selfJudge = protocol(); - const oracleOpenAi = selfJudge.phases[2].arms.find((arm) => arm.id === 'oracle-terra-low')!; - oracleOpenAi.stages[1].provider = 'openai'; - oracleOpenAi.stages[1].model = 'gpt-5.6-terra'; - oracleOpenAi.stages[1].pricingProfileId = 'openai-gpt-5.6-standard-2026-09-05:gpt-5.6-terra'; - expect(() => assertFixedTraceEvaluationProtocol(selfJudge)).toThrow('not provider-independent'); - - const duplicate = protocol(); - duplicate.phases[1].arms.push(structuredClone(duplicate.phases[1].arms[0])); - expect(() => assertFixedTraceEvaluationProtocol(duplicate)).toThrow('Duplicate protocol arm ID'); + it('keeps Terra and Sol as unpriced inert descriptors', () => { + expect(FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES).toEqual([{ provider: 'openai', model: 'gpt-5.6-terra', dispatchable: false, trustedPrice: null }, { provider: 'openai', model: 'gpt-5.6-sol', dispatchable: false, trustedPrice: null }]); + const terra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + terra.phases[1].arms[0].stages[0].provider = 'openai'; terra.phases[1].arms[0].stages[0].model = 'gpt-5.6-terra'; + expect(() => assertFixedTraceEvaluationProtocol(terra)).toThrow('pricing profile does not match'); }); - - it('requires an evaluator-owned manifest before a protocol can become executable evidence', () => { - const current = protocol(); - const fingerprint = fixedTraceEvaluationProtocolFingerprint(current); - expect(() => assertFixedTraceEvaluationProtocolTrusted(current, () => null)).toThrow('Trusted evaluation manifest is unavailable'); - const suites = Object.fromEntries(current.phases.map((phase) => [ - phase.id, - evaluatorOwnedSuite(phase.id, phase.uniqueCaseCount), - ])) as Record; - const traceSuiteSha256ByPhase = Object.fromEntries(current.phases.map((phase) => [ - phase.id, - fixedTraceSuiteSha256(suites[phase.id]), - ])) as Record; - const trusted = { - id: current.trustedManifestId, - protocolFingerprint: fingerprint, - sourceId: 'externally-sealed-addie-v120', - sourceRevision: 'sealed-revision-1', - traceSuiteSha256ByPhase, - tracePackSha256: 'a'.repeat(64), - rawLedgerVersion: 'addie-fixed-trace-raw-ledger-v2', - partitions: Object.fromEntries(current.phases.map((phase) => [phase.id, phase.uniqueCaseCount])), - verifiedAdmissions: ['planning_only', 'requires_verified_hybrid_contract', 'requires_verified_direct_contract'] as const, - }; - expect(assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toBe(trusted); - expect(fixedTraceEvaluationProtocolRunnerBinding( - current, - (id) => id === trusted.id ? trusted : null, - 'router_screen', - suites.router_screen, - )).toEqual({ - trustedManifestId: trusted.id, - protocolFingerprint: fingerprint, - phaseId: 'router_screen', - traceSuite: suites.router_screen, - traceSuiteSha256: traceSuiteSha256ByPhase.router_screen, - }); - const forgedSuite = structuredClone(suites.router_screen); - forgedSuite[0]!.id = 'forged-suite-case'; - expect(() => fixedTraceEvaluationProtocolRunnerBinding( - current, - (id) => id === trusted.id ? trusted : null, - 'router_screen', - forgedSuite, - )).toThrow('does not match trusted manifest'); - trusted.partitions.sealed_final = 37; - expect(() => assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toThrow('count mismatch'); - trusted.partitions.sealed_final = 38; - trusted.traceSuiteSha256ByPhase.router_screen = 'not-a-digest'; - expect(() => assertFixedTraceEvaluationProtocolTrusted(current, (id) => id === trusted.id ? trusted : null)).toThrow('suite hash is unavailable'); + it('rejects reversed, duplicated, direct, smoke-promotion, and fabricated trust', () => { + const reversed = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); reversed.phases.reverse(); + expect(() => assertFixedTraceEvaluationProtocol(reversed)).toThrow('exact required order'); + const direct = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); direct.phases[3].arms[0].architecture = 'direct_bounded_production_shaped'; + expect(() => assertFixedTraceEvaluationProtocol(direct)).toThrow('direct and hybrid'); + const promotional = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; promotional.phases[0].resultUse = 'promotional'; + expect(() => assertFixedTraceEvaluationProtocol(promotional)).toThrow(); + expect(() => assertFixedTraceEvaluationProtocolTrusted(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any)).toThrow('locked'); + expect(() => fixedTraceEvaluationProtocolRunnerBinding(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any, 'bounded_smoke', [])).toThrow('locked'); }); }); diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts index ae32401542..46f6ba4fcd 100644 --- a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts +++ b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts @@ -1,280 +1,41 @@ import { describe, expect, it } from 'vitest'; -import { - FIXED_TRACE_EXPERIMENT_PLAN_VERSION, - FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, - estimateFixedTraceExperiment, - fixedTraceExperimentExecutionOrder, - fixedTraceExperimentPartitionAudit, - fixedTraceCandidatePlanFingerprint, - fixedTraceDevelopmentSelectionArtifact, - consumeFixedTraceHoldoutFinalization, - fixedTraceExperimentPlanFingerprint, - fixedTraceExperimentRunnerBinding, - fixedTraceTrustedManifestFingerprint, - assertFixedTraceRawAuditableLedger, - type FixedTraceExperimentPlan, - type FixedTracePlannedStage, -} from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; -import { - FIXED_TRACE_PARTITION_MANIFEST, - FIXED_TRACE_PARTITION_MANIFEST_SHA256, - FIXED_TRACE_PARTITION_MANIFEST_VERSION, -} from '../../../src/addie/eval/fixed-trace-partition.js'; +import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; +import { FIXED_TRACE_PARTITION_MANIFEST, FIXED_TRACE_PARTITION_MANIFEST_SHA256, FIXED_TRACE_PARTITION_MANIFEST_VERSION } from '../../../src/addie/eval/fixed-trace-partition.js'; import { CLAUDE_PRICING_VERSION } from '../../../src/addie/claude-pricing.js'; -import { - GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - OPENAI_GPT_5_6_PRICING_VERSION, -} from '../../../src/addie/model-cost-pricing.js'; import { CODE_VERSION } from '../../../src/addie/config-version.js'; -import { canonicalFixedTraceToolDefinitions } from '../../../src/addie/eval/fixed-trace-tools.js'; -import { - FIXED_TRACE_STAGE_CONTROL_VERSION, - FIXED_TRACE_SUITE, - fixedTraceSuiteSha256, -} from '../../../src/addie/eval/fixed-trace-suite.js'; -import { fixedTraceToolSchemaSha256 } from '../../../src/addie/eval/fixed-trace-runner.js'; +import { FIXED_TRACE_STAGE_CONTROL_VERSION, FIXED_TRACE_SUITE } from '../../../src/addie/eval/fixed-trace-suite.js'; const HASH = 'a'.repeat(64); - -function trustedSuite(ids: readonly string[]) { - const traceSuite = FIXED_TRACE_SUITE.filter((trace) => ids.includes(trace.id)); - const fixtureNames = new Set(traceSuite.flatMap((trace) => trace.toolFixtures.map((fixture) => fixture.name))); - const toolDefinitions = canonicalFixedTraceToolDefinitions().filter((definition) => fixtureNames.has(definition.name)); - return { - traceSuite, - traceSuiteSha256: fixedTraceSuiteSha256(traceSuite), - toolDefinitions, - toolSchemaSha256: fixedTraceToolSchemaSha256(traceSuite, toolDefinitions), - toolDefinitionProvenance: 'fixture_local' as const, - }; -} - -const trustedManifest = { - id: 'trusted-synthetic-v1', - sourceId: 'fixed-trace-synthetic-corpus', - sourceRevision: 'addie-fixed-traces-v32', - sourceBundleSha256: HASH, - promptConfigVersion: HASH, - suites: { - development: trustedSuite(FIXED_TRACE_PARTITION_MANIFEST.development), - holdout: trustedSuite(FIXED_TRACE_PARTITION_MANIFEST.holdout), - }, - partitionManifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, - rawLedgerVersion: 'addie-fixed-trace-raw-ledger-v1' as const, - gitCommit: 'a'.repeat(40), - gitDirty: false, - addieCodeVersion: CODE_VERSION, - stageControlVersion: FIXED_TRACE_STAGE_CONTROL_VERSION, - providerDegradationInjectionEnabled: true, -}; -const resolver = (id: string) => id === trustedManifest.id ? trustedManifest : null; - -function stage( - provider: FixedTracePlannedStage['provider'], - model: string, - pricingVersion: string, - maxIterations = 1, - traceIds = FIXED_TRACE_PARTITION_MANIFEST.development, -): FixedTracePlannedStage { - return { - provider, - model, - pricingVersion, - reasoningEffort: 'none', - maxOutputTokens: 10, - timeoutMs: 1_000, - maxIterations, - transportRetries: 0, - samplingMode: 'provider_no_sampling_control', - temperature: null, - cacheMode: 'disabled', - requestBounds: { inputBytesByTrace: Object.fromEntries(traceIds.map((id) => [id, Array(maxIterations).fill(100)])) }, - }; -} - -function plan(overrides: Partial = {}): FixedTraceExperimentPlan { - const selected = overrides.partition?.selected ?? 'development'; - const suite = trustedManifest.suites[selected]; - const router = stage('openai', 'gpt-5.6-luna', OPENAI_GPT_5_6_PRICING_VERSION); - const generation = stage('openai', 'gpt-5.6-terra', OPENAI_GPT_5_6_PRICING_VERSION, 2); - return { - version: FIXED_TRACE_EXPERIMENT_PLAN_VERSION, - id: 'matrix-v1', - trustedManifestId: trustedManifest.id, - sourceId: trustedManifest.sourceId, - sourceRevision: trustedManifest.sourceRevision, - pricingAsOf: '2026-09-05T12:00:00.000Z', - sourceBundleSha256: HASH, - gitCommit: trustedManifest.gitCommit, - gitDirty: trustedManifest.gitDirty, - addieCodeVersion: trustedManifest.addieCodeVersion, - traceSuiteSha256: suite.traceSuiteSha256, - promptConfigVersion: HASH, - toolSchemaSha256: suite.toolSchemaSha256, - toolDefinitionProvenance: suite.toolDefinitionProvenance, - stageControlVersion: trustedManifest.stageControlVersion, - providerDegradationInjectionEnabled: trustedManifest.providerDegradationInjectionEnabled, - partition: { - manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, - manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, - selected: 'development', - }, - ordering: { seed: 'recorded-seed-v1' }, - budgets: { candidateCeilingUsd: 1, judgeCeilingUsd: 1 }, - arms: [{ - id: 'terra-finalist-r1', - architecture: 'two_stage_llm_router', - screeningStage: 'deployable_finalist', - repetitionIndex: 1, - router, - generation, - judges: [ - { ...stage('anthropic', 'claude-sonnet-5', CLAUDE_PRICING_VERSION), blinded: true }, - { ...stage('google', 'gemini-3.7-flash', GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION), blinded: true }, - ], - }], - ...overrides, - }; +function plan(): FixedTraceExperimentPlan { + const inputBytesByTrace = Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.development.map((id) => [id, [100]])); + return { version: FIXED_TRACE_EXPERIMENT_PLAN_VERSION, id: 'offline-v1', trustedManifestId: 'unissued', sourceId: 'fixture', sourceRevision: 'v1', pricingAsOf: '2026-09-05T12:00:00.000Z', sourceBundleSha256: HASH, gitCommit: 'a'.repeat(40), gitDirty: false, addieCodeVersion: CODE_VERSION, stageControlVersion: FIXED_TRACE_STAGE_CONTROL_VERSION, traceSuiteSha256: HASH, promptConfigVersion: HASH, toolSchemaSha256: HASH, toolDefinitionProvenance: 'fixture_local', providerDegradationInjectionEnabled: true, partition: { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, selected: 'development' }, ordering: { seed: 'seed' }, budgets: { candidateCeilingUsd: 1, judgeCeilingUsd: 1 }, arms: [{ id: 'router-r1', architecture: 'two_stage_llm_router', screeningStage: 'router_only_screen', repetitionIndex: 1, router: { provider: 'anthropic', model: 'claude-haiku-4-5', reasoningEffort: 'provider_default', pricingVersion: CLAUDE_PRICING_VERSION, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0, samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', requestBounds: { inputBytesByTrace } } }] }; } -describe('fixed-trace experiment plan', () => { - it('estimates a pure conservative ceiling with independently budgeted judges', () => { - const estimate = estimateFixedTraceExperiment(plan(), resolver); - expect(estimate).toMatchObject({ diagnosticOnly: true, comparisonEligible: false }); - expect(estimate.expectedSpendUsd).toBeNull(); - expect(estimate.candidate.expectedSpendUsd).toBeNull(); - expect(estimate.judges.expectedSpendUsd).toBeNull(); - expect(estimate.candidate.reservations.map((item) => item.stage)).toEqual(['router', 'generation']); - expect(estimate.judges.reservations.map((item) => item.stage)).toEqual(['judge', 'judge']); - expect(estimate.totalCeilingUsd).toBe(estimate.candidate.ceilingUsd + estimate.judges.ceilingUsd); - expect(estimate.candidate.reservations[1]).toMatchObject({ requests: 48, inputBytes: 4_800, outputTokens: 480 }); - expect(estimate.budgetIdentitySha256).toMatch(/^[a-f0-9]{64}$/); - }); - - it('passes the frozen evaluator-owned suite and provenance unchanged to a future runner', () => { - const current = plan(); - const binding = fixedTraceExperimentRunnerBinding(current, resolver, 'terra-finalist-r1'); - expect(binding).toMatchObject({ - runId: 'matrix-v1:terra-finalist-r1:r1', - traceSuiteSha256: current.traceSuiteSha256, - toolDefinitionProvenance: 'fixture_local', - providerDegradationInjectionEnabled: true, - }); - expect(Object.isFrozen(binding.traceSuite)).toBe(true); - expect(Object.isFrozen(binding.traceSuite[0])).toBe(true); - expect(Object.isFrozen(binding.toolDefinitions)).toBe(true); - expect(Object.isFrozen(binding.toolDefinitions[0])).toBe(true); - - const forged = structuredClone(trustedManifest); - forged.suites.development.traceSuite[0]!.id = 'forged-trace'; - expect(() => fixedTraceExperimentRunnerBinding(current, (id) => id === forged.id ? forged : null, 'terra-finalist-r1')) - .toThrow('suite does not exactly bind'); - }); - - it('fails closed for spoofed manifests, unknown pricing, and missing request bounds', () => { - expect(() => fixedTraceExperimentPlanFingerprint(plan({ - partition: { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: HASH, selected: 'development' }, - }), resolver)).toThrow('uncommitted fixed-trace partition manifest'); - const unknown = plan(); - unknown.arms[0].router!.pricingVersion = 'price-i-made-up'; - expect(() => fixedTraceExperimentPlanFingerprint(unknown, resolver)).toThrow('Unavailable immutable pricing'); - const missing = plan(); - delete missing.arms[0].router!.requestBounds.inputBytesByTrace['surface-channel-chatter']; - expect(() => fixedTraceExperimentPlanFingerprint(missing, resolver)).toThrow('exact bounds'); - }); - - it('keeps holdout locked unless an explicit versioned finalization gate is present', () => { - const holdout = plan({ - partition: { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, selected: 'holdout' }, - }); - for (const item of holdout.arms) { - if (item.router) item.router.requestBounds = { inputBytesByTrace: Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100]])) }; - if (item.generation) item.generation.requestBounds = { inputBytesByTrace: Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100, 100]])) }; - for (const judge of item.judges ?? []) judge.requestBounds = { inputBytesByTrace: Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100]])) }; - } - expect(() => fixedTraceExperimentPlanFingerprint(holdout, resolver)).toThrow('Holdout is locked'); - holdout.partition.finalizationGate = { version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, recordId: 'finalization-1' }; - const finalization = { - id: 'finalization-1', version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, - trustedManifestId: trustedManifest.id, frozenCandidatePlanFingerprint: fixedTraceCandidatePlanFingerprint(holdout), - consumed: false, tracePackVisibility: 'repository_visible' as const, - }; - const finalizationResolver = (id: string) => id === finalization.id ? finalization : null; - expect(fixedTraceExperimentPartitionAudit(holdout, resolver, finalizationResolver)).toMatchObject({ - selected: 'holdout', manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, - }); - expect(() => fixedTraceDevelopmentSelectionArtifact(holdout, resolver)).toThrow('finalization record'); - let consumed = false; - consumeFixedTraceHoldoutFinalization(holdout, resolver, finalizationResolver, (id, fingerprint) => { - consumed = id === finalization.id && fingerprint === finalization.frozenCandidatePlanFingerprint; - return consumed; - }); - expect(consumed).toBe(true); - holdout.arms[0].generation!.maxOutputTokens++; - expect(() => fixedTraceExperimentPartitionAudit(holdout, resolver, finalizationResolver)).toThrow('frozen candidate plan'); - holdout.arms[0].generation!.maxOutputTokens--; - finalization.consumed = true; - expect(() => fixedTraceExperimentPartitionAudit(holdout, resolver, finalizationResolver)).toThrow('already been consumed'); - }); - - it('rejects candidate self-judging, insufficient judges, duplicate arms, and unimplemented architectures', () => { - const selfJudge = plan(); - selfJudge.arms[0].judges![0] = { ...stage('openai', 'gpt-5.6-sol', OPENAI_GPT_5_6_PRICING_VERSION), blinded: true }; - expect(() => fixedTraceExperimentPlanFingerprint(selfJudge, resolver)).toThrow('not provider-independent'); - const duplicate = plan(); - duplicate.arms = [...duplicate.arms, structuredClone(duplicate.arms[0])]; - expect(() => fixedTraceExperimentPlanFingerprint(duplicate, resolver)).toThrow('Duplicate experiment arm ID'); - const direct = plan(); - direct.arms[0].architecture = 'direct_generation'; - expect(() => fixedTraceExperimentPlanFingerprint(direct, resolver)).toThrow('inadmissible'); - const hybrid = plan(); - hybrid.arms[0].architecture = 'hybrid_generation'; - expect(() => fixedTraceExperimentPlanFingerprint(hybrid, resolver)).toThrow('inadmissible'); - }); - - it('rejects a ceiling that under-reserves either candidate or judge work', () => { - const candidate = plan({ budgets: { candidateCeilingUsd: 0.000001, judgeCeilingUsd: 1 } }); - expect(() => estimateFixedTraceExperiment(candidate, resolver)).toThrow('Candidate worst-case'); - const judges = plan({ budgets: { candidateCeilingUsd: 1, judgeCeilingUsd: 0.000001 } }); - expect(() => estimateFixedTraceExperiment(judges, resolver)).toThrow('Judge worst-case'); - }); - - it('records a seed-based order and fingerprints every material control', () => { - const repeated = plan(); - repeated.arms = ['luna', 'terra', 'sol'].map((name, index) => ({ - id: `${name}-router-r${index + 1}`, - architecture: 'two_stage_llm_router' as const, - screeningStage: 'router_only_screen' as const, - repetitionIndex: index + 1, - router: stage('openai', `gpt-5.6-${name}`, OPENAI_GPT_5_6_PRICING_VERSION), - })); - const first = fixedTraceExperimentExecutionOrder(repeated, resolver); - expect(first).toEqual(fixedTraceExperimentExecutionOrder(structuredClone(repeated), resolver)); - repeated.ordering.seed = 'a different recorded seed'; - expect(fixedTraceExperimentExecutionOrder(repeated, resolver)).not.toEqual(first); - const baseline = fixedTraceExperimentPlanFingerprint(plan(), resolver); - const changed = plan(); - changed.arms[0].generation!.timeoutMs++; - expect(fixedTraceExperimentPlanFingerprint(changed, resolver)).not.toBe(baseline); - expect(fixedTraceDevelopmentSelectionArtifact(plan(), resolver)).toMatchObject({ - holdoutMetricsIncluded: false, - blindingLimitation: 'execution_locked_repository_visible_not_secret_holdout', - }); - }); - - it('requires externally resolved trusted inputs and raw, identity-complete ledger entries', () => { - expect(() => estimateFixedTraceExperiment(plan(), () => null)).toThrow('Trusted fixed-trace manifest is unavailable'); +describe('fixed-trace experiment plan offline boundary', () => { + it('is diagnostic only and has no trust or dispatch lock', () => { + expect(validateFixedTraceExperimentPlanOffline(plan())).toMatchObject({ diagnosticOnly: true, comparisonEligible: false, dispatchable: false, trustedLock: false }); + }); + it('rejects inherited, accessor, proxy, extra-field, and unpriced Terra input', () => { + const inherited = Object.assign(Object.create(plan()), { version: FIXED_TRACE_EXPERIMENT_PLAN_VERSION }); + expect(() => validateFixedTraceExperimentPlanOffline(inherited)).toThrow('plain object'); + const getter = plan(); Object.defineProperty(getter, 'id', { enumerable: true, get: () => 'getter' }); + expect(() => validateFixedTraceExperimentPlanOffline(getter)).toThrow('own enumerable data'); + expect(() => validateFixedTraceExperimentPlanOffline(new Proxy(plan(), {}))).toThrow('Proxy'); + const extra = plan() as FixedTraceExperimentPlan & { extra: boolean }; extra.extra = true; + expect(() => validateFixedTraceExperimentPlanOffline(extra)).toThrow('unknown'); + const terra = plan(); terra.arms[0].router!.model = 'gpt-5.6-terra'; + expect(() => validateFixedTraceExperimentPlanOffline(terra)).toThrow('Unavailable immutable pricing'); + }); + it('requires exact ledger sequence, tools, and offline provider resolution', () => { const current = plan(); - const fingerprint = fixedTraceExperimentPlanFingerprint(current, resolver); - const ledger = { - version: 'addie-fixed-trace-raw-ledger-v1' as const, - trustedManifestSha256: 'b'.repeat(64), - planFingerprint: fingerprint, - budgetIdentitySha256: estimateFixedTraceExperiment(current, resolver).budgetIdentitySha256, - entries: [], - }; - expect(() => assertFixedTraceRawAuditableLedger(current, resolver, ledger, () => null)).toThrow('trusted manifest mismatch'); - ledger.trustedManifestSha256 = fixedTraceTrustedManifestFingerprint(trustedManifest); - expect(() => assertFixedTraceRawAuditableLedger(current, resolver, ledger, () => null)).toThrow('lacks complete planned-stage coverage'); + const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); + const ledger = { version: 'addie-fixed-trace-raw-ledger-v1' as const, trustedManifestSha256: HASH, planFingerprint: HASH, budgetIdentitySha256: HASH, entries }; + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).not.toThrow(); + ledger.entries[1].sequence = 1; + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).toThrow('sequence'); + ledger.entries[1].sequence = 2; ledger.entries[0].exactToolNames = ['tampered']; + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).toThrow('tool names'); + ledger.entries[0].exactToolNames = FIXED_TRACE_SUITE.find((item) => item.id === ledger.entries[0].traceId)!.toolFixtures.map((fixture) => fixture.name); ledger.entries[0].returnedProvider = 'google'; ledger.entries[0].returnedModel = 'gemini-3.7-flash'; + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).toThrow('dispatch, response'); }); }); diff --git a/server/tests/unit/addie/model-provider-openai-google.test.ts b/server/tests/unit/addie/model-provider-openai-google.test.ts index 9ce5bc3c71..d5519d4ed5 100644 --- a/server/tests/unit/addie/model-provider-openai-google.test.ts +++ b/server/tests/unit/addie/model-provider-openai-google.test.ts @@ -3,7 +3,6 @@ import type { Response } from 'openai/resources/responses/responses'; import type { GenerateContentResponse } from '@google/genai'; import { collectModelResponse } from '../../../src/addie/model-providers/events.js'; import { - OPENAI_FIXED_TRACE_MODELS, OPENAI_ROUTER_MODEL, OpenAIResponsesProvider, normalizeOpenAIResponse, @@ -90,11 +89,11 @@ function googleResponse(overrides: Record = {}): GenerateConten } describe('OpenAIResponsesProvider', () => { - it('accepts only the explicit fixed-trace OpenAI model IDs', () => { + it('keeps Terra and Sol outside the production OpenAI dispatch boundary', () => { const provider = new OpenAIResponsesProvider('unused', {} as OpenAIResponsesTransport); - for (const model of OPENAI_FIXED_TRACE_MODELS) { - expect(provider.prepare(request(model)).providerRequest).toMatchObject({ model }); - } + expect(provider.prepare(request(OPENAI_ROUTER_MODEL)).providerRequest).toMatchObject({ model: OPENAI_ROUTER_MODEL }); + expect(() => provider.prepare(request('gpt-5.6-terra'))).toThrow('Unsupported OpenAI router model'); + expect(() => provider.prepare(request('gpt-5.6-sol'))).toThrow('Unsupported OpenAI router model'); expect(() => provider.prepare(request('gpt-5.6-terra-20260905'))).toThrow('Unsupported OpenAI router model'); }); From d0b6beb4c7525f216dc0ee47239719de578e78da Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 17:44:01 +0000 Subject: [PATCH 06/16] fix(addie): harden offline evaluation boundaries --- .../addie/eval/fixed-trace-diagnostic-run.ts | 51 ++++--- .../eval/fixed-trace-evaluation-protocol.ts | 105 ++++++++------ .../addie/eval/fixed-trace-experiment-plan.ts | 128 +++++++----------- .../addie/eval/fixed-trace-safe-snapshot.ts | 69 ++++++++++ .../training-agent/reporting-reliability.ts | 4 +- .../tests/manual/fixed-trace-provider-eval.ts | 2 +- .../fixed-trace-evaluation-protocol.test.ts | 58 +++++++- .../addie/fixed-trace-experiment-plan.test.ts | 31 ++++- 8 files changed, 293 insertions(+), 155 deletions(-) create mode 100644 server/src/addie/eval/fixed-trace-safe-snapshot.ts diff --git a/server/src/addie/eval/fixed-trace-diagnostic-run.ts b/server/src/addie/eval/fixed-trace-diagnostic-run.ts index b319cad9d8..1e29de092e 100644 --- a/server/src/addie/eval/fixed-trace-diagnostic-run.ts +++ b/server/src/addie/eval/fixed-trace-diagnostic-run.ts @@ -23,6 +23,8 @@ import { fixedTraceResponsePricingPolicy, isTrustedBudgetedFixedTraceProvider, } from './fixed-trace-budget.js'; +import { types } from 'node:util'; +import { snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js'; export interface FixedTraceDiagnosticProviderPlan { readonly name: string; @@ -64,6 +66,17 @@ function ownDataProperty(source: unknown, name: string, owner: string): unknown return descriptor.value; } +function assertClosedOwnDataRecord(source: unknown, fields: readonly string[], owner: string): void { + if (typeof source !== 'object' || source === null || types.isProxy(source) || Object.getPrototypeOf(source) !== Object.prototype) { + throw new Error(`Fixed trace diagnostic ${owner} must be a plain non-Proxy object`); + } + const keys = Reflect.ownKeys(source); + if (keys.length !== fields.length || keys.some((key) => typeof key !== 'string' || !fields.includes(key))) { + throw new Error(`Fixed trace diagnostic ${owner} must contain exactly its approved fields`); + } + for (const field of fields) ownDataProperty(source, field, owner); +} + const DIAGNOSTIC_PRICING_FIELDS = [ 'profileId', 'inputUsdPerMillionTokens', @@ -76,19 +89,7 @@ const DIAGNOSTIC_PRICING_FIELDS = [ ] as const; function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing { - const prototype = typeof pricing === 'object' && pricing !== null - ? Object.getPrototypeOf(pricing) - : null; - if ( - typeof pricing !== 'object' - || pricing === null - || (prototype !== Object.prototype && prototype !== null) - ) throw new Error(`Fixed trace diagnostic ${owner} must be a plain pricing object`); - const keys = Reflect.ownKeys(pricing); - if ( - keys.length !== DIAGNOSTIC_PRICING_FIELDS.length - || keys.some((key) => typeof key !== 'string' || !DIAGNOSTIC_PRICING_FIELDS.includes(key as typeof DIAGNOSTIC_PRICING_FIELDS[number])) - ) throw new Error(`Fixed trace diagnostic ${owner} must contain only approved pricing fields`); + assertClosedOwnDataRecord(pricing, DIAGNOSTIC_PRICING_FIELDS, owner); // Structured cloning calls nested getters. Copy each approved data // descriptor instead, so a price cannot change between validation and use. return Object.freeze({ @@ -106,6 +107,10 @@ function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing { function snapshotStageConfig(config: unknown, owner: string): FixedTraceProviderStageConfig { // Read each untrusted stage property exactly once. Later checks use only // this detached plain object, never a caller-controlled getter or proxy. + assertClosedOwnDataRecord(config, [ + 'provider', 'model', 'reasoningEffort', 'maxOutputTokens', 'timeoutMs', + 'maxIterations', 'transportRetries', 'samplingMode', 'temperature', 'pricing', + ], owner); const provider = ownDataProperty(config, 'provider', owner); const model = ownDataProperty(config, 'model', owner); const reasoningEffort = ownDataProperty(config, 'reasoningEffort', owner); @@ -133,12 +138,7 @@ function snapshotStageConfig(config: unknown, owner: string): FixedTraceProvider function snapshotBaseConfig( config: FixedTraceDiagnosticArtifactOptions['baseConfig'], ): FixedTraceDiagnosticArtifactOptions['baseConfig'] { - const { traceSuite, toolDefinitions, ...serializable } = config; - return Object.freeze({ - ...structuredClone(serializable), - traceSuite: deepFreeze(structuredClone(traceSuite)), - toolDefinitions: deepFreeze(structuredClone(toolDefinitions)), - }); + return snapshotFixedTraceJson(config, 'fixed trace diagnostic base config') as FixedTraceDiagnosticArtifactOptions['baseConfig']; } function snapshotPlans( @@ -148,6 +148,19 @@ function snapshotPlans( if (!Array.isArray(suppliedPlans) || suppliedPlans.length === 0) { throw new Error('Fixed trace diagnostic run requires one or more provider plans'); } + if (types.isProxy(suppliedPlans) || Object.getPrototypeOf(suppliedPlans) !== Array.prototype || Object.getOwnPropertySymbols(suppliedPlans).length !== 0) { + throw new Error('Fixed trace diagnostic provider plans must be a plain non-Proxy array'); + } + const planDescriptors = Object.getOwnPropertyDescriptors(suppliedPlans); + for (const key of Object.keys(planDescriptors)) { + if (key === 'length') continue; + if (!/^(0|[1-9][0-9]*)$/.test(key) || !('value' in planDescriptors[key]!) || !planDescriptors[key]!.enumerable) { + throw new Error('Fixed trace diagnostic provider plans contain an accessor or extra property'); + } + } + for (const [index, suppliedPlan] of suppliedPlans.entries()) { + assertClosedOwnDataRecord(suppliedPlan, ['name', 'router', 'generation'], `provider plan ${index}`); + } const plans = Object.freeze(suppliedPlans.map((suppliedPlan, index) => Object.freeze({ // Do not validate while reading: a plan accessor must not be able to // return one identity for validation and another for execution. diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index e733f54cf1..b6f7fff13b 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -9,10 +9,10 @@ import { validateFixedTracePricing, } from './fixed-trace-budget.js'; import { - fixedTraceSuiteSha256, type FixedTraceCase, type FixedTracePricing, } from './fixed-trace-suite.js'; +import { deepFreezeFixedTrace, snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js'; /** * A planning-only contract. It has no dispatcher and is deliberately unable @@ -26,8 +26,7 @@ export type FixedTraceProtocolPhaseId = | 'router_screen' | 'oracle_generator_ceiling' | 'deployable_architecture' - | 'controlled_tuning' - | 'sealed_final'; + | 'controlled_tuning'; export type FixedTraceProtocolArchitecture = | 'two_stage_llm_router' @@ -143,6 +142,13 @@ export interface FixedTraceEvaluationProtocol { trustedManifestId: string; pricingAsOf: string; contingencyBasisPoints: number; + /** A planning deficit only; it is not an executable or authenticated phase. */ + unavailableFinalTarget: { + availability: 'unavailable'; + uniqueCaseCount: number; + repetitions: number; + missingCaseCount: number; + }; phases: readonly FixedTraceProtocolPhase[]; } @@ -216,7 +222,7 @@ export interface FixedTraceProtocolEstimate { stages: readonly FixedTraceProtocolStageEstimate[]; phases: readonly FixedTraceProtocolPhaseEstimate[]; screening: { candidateCeilingUsd: number; judgeCeilingUsd: number; totalCeilingUsd: number }; - finalConfirmation: { candidateCeilingUsd: number; judgeCeilingUsd: number; totalCeilingUsd: number }; + unavailableFinalTarget: FixedTraceEvaluationProtocol['unavailableFinalTarget']; candidateCeilingUsd: number; judgeCeilingUsd: number; contingencyUsd: number; @@ -245,6 +251,14 @@ function positiveInteger(value: number, label: string): void { if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`); } +function assertExactKeys(value: object, keys: readonly string[], label: string): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new Error(`${label} has unknown, missing, or inherited fields`); + } +} + function pricing(profileId: string, pricingAsOf: string): FixedTraceProtocolPricingProfile { const profile = FIXED_TRACE_PROTOCOL_PRICING.find((candidate) => candidate.profileId === profileId); if (!profile) throw new Error(`Unavailable immutable pricing profile: ${profileId}`); @@ -257,6 +271,11 @@ function pricing(profileId: string, pricingAsOf: string): FixedTraceProtocolPric } function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: string): FixedTraceProtocolPricingProfile { + assertExactKeys(stage, [ + 'role', 'provider', 'model', 'reasoningEffort', 'pricingProfileId', + 'maxInputTokensPerInvocation', 'maxOutputTokensPerInvocation', 'timeoutMs', + 'maxInvocationsPerCase', 'transportRetries', 'samplingMode', 'temperature', 'cacheMode', + ], label); positiveInteger(stage.maxInputTokensPerInvocation, `${label}.maxInputTokensPerInvocation`); positiveInteger(stage.maxOutputTokensPerInvocation, `${label}.maxOutputTokensPerInvocation`); positiveInteger(stage.timeoutMs, `${label}.timeoutMs`); @@ -273,11 +292,8 @@ function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: return resolved; } -function candidateProviders(arm: FixedTraceProtocolArm): Set { - return new Set(arm.stages.filter((stage) => stage.role !== 'judge').map((stage) => stage.provider)); -} - function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, pricingAsOf: string): void { + assertExactKeys(arm, ['id', 'architecture', 'admission', 'stages'], `protocol arm`); if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid protocol arm ID: ${arm.id}`); const routers = arm.stages.filter((stage) => stage.role === 'router'); const generations = arm.stages.filter((stage) => stage.role === 'generation'); @@ -302,13 +318,23 @@ function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, p if (judges.length !== 0) throw new Error(`${arm.id} judges are blocked in the diagnostic-only protocol`); } -/** Fingerprints every material execution and budget control; no resolver is trusted here. */ +function validatedProtocolSnapshot(protocol: FixedTraceEvaluationProtocol): FixedTraceEvaluationProtocol { + const snapshot = snapshotFixedTraceJson(protocol, 'evaluation protocol') as FixedTraceEvaluationProtocol; + assertFixedTraceEvaluationProtocolStructure(snapshot); + return snapshot; +} + +/** Fingerprints the exact detached projection which passed all protocol checks. */ export function fixedTraceEvaluationProtocolFingerprint(protocol: FixedTraceEvaluationProtocol): string { - return sha256(protocol); + return sha256(validatedProtocolSnapshot(protocol)); } /** Validate the planning projection without loading traces, credentials, or providers. */ -export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): void { +function assertFixedTraceEvaluationProtocolStructure(protocol: FixedTraceEvaluationProtocol): void { + assertExactKeys(protocol, [ + 'version', 'id', 'trustedManifestId', 'pricingAsOf', 'contingencyBasisPoints', + 'unavailableFinalTarget', 'phases', + ], 'evaluation protocol'); if (protocol.version !== FIXED_TRACE_EVALUATION_PROTOCOL_VERSION || !protocol.id.trim() || !protocol.trustedManifestId.trim()) { throw new Error('Unsupported or incomplete fixed-trace evaluation protocol'); } @@ -317,11 +343,19 @@ export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluatio } const phaseIds = new Set(); const armIds = new Set(); - const requiredOrder = ['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning', 'sealed_final'] as const; + assertExactKeys(protocol.unavailableFinalTarget, ['availability', 'uniqueCaseCount', 'repetitions', 'missingCaseCount'], 'evaluation protocol.unavailableFinalTarget'); + if (protocol.unavailableFinalTarget.availability !== 'unavailable' + || protocol.unavailableFinalTarget.uniqueCaseCount !== 38 + || protocol.unavailableFinalTarget.repetitions !== 3 + || protocol.unavailableFinalTarget.missingCaseCount !== 38) { + throw new Error('Protocol unavailable final target is invalid'); + } + const requiredOrder = ['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning'] as const; if (protocol.phases.length !== requiredOrder.length || protocol.phases.some((phase, index) => phase.id !== requiredOrder[index])) { throw new Error('Protocol phases must use the exact required order'); } for (const phase of protocol.phases) { + assertExactKeys(phase, ['id', 'uniqueCaseCount', 'repetitions', 'resultUse', 'arms'], 'protocol phase'); if (phaseIds.has(phase.id)) throw new Error(`Duplicate protocol phase: ${phase.id}`); phaseIds.add(phase.id); positiveInteger(phase.uniqueCaseCount, `${phase.id}.uniqueCaseCount`); @@ -337,6 +371,10 @@ export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluatio for (const required of requiredOrder) if (!phaseIds.has(required)) throw new Error(`Protocol is missing required phase: ${required}`); } +export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): void { + void validatedProtocolSnapshot(protocol); +} + /** * Future execution must supply evaluator-owned data. This check intentionally * does not make a JSON protocol file trusted by comparing it to itself. @@ -405,10 +443,10 @@ function stageEstimate( * observed tokenization and tool-loop length are deliberately not guessed. */ export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): FixedTraceProtocolEstimate { - assertFixedTraceEvaluationProtocol(protocol); - const stages = protocol.phases.flatMap((phase) => phase.arms.flatMap((arm) => - arm.stages.map((stage) => stageEstimate(phase, arm, stage, protocol.pricingAsOf)))); - const phases = protocol.phases.map((phase) => { + const snapshot = validatedProtocolSnapshot(protocol); + const stages = snapshot.phases.flatMap((phase) => phase.arms.flatMap((arm) => + arm.stages.map((stage) => stageEstimate(phase, arm, stage, snapshot.pricingAsOf)))); + const phases = snapshot.phases.map((phase) => { const entries = stages.filter((entry) => entry.phaseId === phase.id); const candidate = entries.filter((entry) => entry.role !== 'judge'); const judges = entries.filter((entry) => entry.role === 'judge'); @@ -425,24 +463,22 @@ export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluat totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd, }); }); - const screeningPhases = phases.filter((phase) => phase.phaseId !== 'sealed_final'); - const finalPhase = phases.find((phase) => phase.phaseId === 'sealed_final')!; const candidateCeilingUsd = phases.reduce((total, phase) => total + phase.candidateCeilingUsd, 0); const judgeCeilingUsd = phases.reduce((total, phase) => total + phase.judgeCeilingUsd, 0); - const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * protocol.contingencyBasisPoints / 10_000; + const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * snapshot.contingencyBasisPoints / 10_000; const summarize = (source: readonly FixedTraceProtocolPhaseEstimate[]) => Object.freeze({ candidateCeilingUsd: source.reduce((total, phase) => total + phase.candidateCeilingUsd, 0), judgeCeilingUsd: source.reduce((total, phase) => total + phase.judgeCeilingUsd, 0), totalCeilingUsd: source.reduce((total, phase) => total + phase.totalCeilingUsd, 0), }); return Object.freeze({ - protocolFingerprint: fixedTraceEvaluationProtocolFingerprint(protocol), + protocolFingerprint: sha256(snapshot), dispatchable: false, expectedSpendUsd: null, stages: Object.freeze(stages), phases: Object.freeze(phases), - screening: summarize(screeningPhases), - finalConfirmation: summarize([finalPhase]), + screening: summarize(phases), + unavailableFinalTarget: snapshot.unavailableFinalTarget, candidateCeilingUsd, judgeCeilingUsd, contingencyUsd, @@ -474,18 +510,6 @@ const generation = ( samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', }); -const judge = ( - provider: ModelProviderId, - model: string, - reasoningEffort: ModelReasoningEffort, - pricingProfileId: string, -): FixedTraceProtocolStage => ({ - role: 'judge', provider, model, reasoningEffort, pricingProfileId, - maxInputTokensPerInvocation: 8_192, maxOutputTokensPerInvocation: 600, - timeoutMs: 60_000, maxInvocationsPerCase: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', -}); - const PRICE = Object.freeze({ haiku: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, sonnet: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, @@ -499,12 +523,15 @@ export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ ]); /** A closed, diagnostic-only projection with no promotion or execution path. */ -export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = Object.freeze({ +export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = deepFreezeFixedTrace({ version: FIXED_TRACE_EVALUATION_PROTOCOL_VERSION, id: 'addie-6842-6846-staged-v1', trustedManifestId: 'externally-owned-addie-fixed-trace-v120', pricingAsOf: '2026-09-05T12:00:00.000Z', contingencyBasisPoints: 0, + unavailableFinalTarget: { + availability: 'unavailable', uniqueCaseCount: 38, repetitions: 3, missingCaseCount: 38, + }, phases: Object.freeze([ Object.freeze({ id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, resultUse: 'diagnostic_only', @@ -545,13 +572,5 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto ]) }), ]), }), - Object.freeze({ - id: 'sealed_final', uniqueCaseCount: 38, repetitions: 3, resultUse: 'diagnostic_only', - arms: Object.freeze([ - Object.freeze({ id: 'final-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - ]) }), - ]), - }), ]), }); diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts index b14c985c50..63cdd0c7c6 100644 --- a/server/src/addie/eval/fixed-trace-experiment-plan.ts +++ b/server/src/addie/eval/fixed-trace-experiment-plan.ts @@ -18,6 +18,7 @@ import { type FixedTraceCase, } from './fixed-trace-suite.js'; import type { FixedTraceToolDefinitionProvenance } from './fixed-trace-architecture.js'; +import { deepFreezeFixedTrace, snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js'; /** A versioned, network-free admission contract for fixed-trace experiments. */ export const FIXED_TRACE_EXPERIMENT_PLAN_VERSION = 'addie-fixed-trace-experiment-plan-v1' as const; @@ -172,7 +173,7 @@ export type FixedTraceTrustedManifestResolver = (id: string) => FixedTraceTruste /** Stable identity for a resolver-owned manifest, used by the raw ledger. */ export function fixedTraceTrustedManifestFingerprint(manifest: FixedTraceTrustedManifest): string { - return sha256(manifest); + return sha256(snapshotFixedTraceJson(manifest, 'trusted manifest')); } /** @@ -214,10 +215,14 @@ export type FixedTraceHoldoutFinalizationConsumer = (id: string, frozenCandidate export interface FixedTraceRawLedgerEntry { sequence: number; + /** The plan-controlled stage grouping; it cannot be supplied out of order. */ + phaseId: FixedTraceScreeningStage; armId: string; repetitionIndex: number; traceId: string; stage: 'router' | 'generation' | 'judge'; + /** One ledger entry represents one configured stage invocation envelope. */ + callIndex: 1; dispatched: boolean; requestedProvider: ModelProviderId | null; requestedModel: string | null; @@ -291,54 +296,6 @@ export interface FixedTraceOfflinePlanValidation { planFingerprint: string; } -/** - * Copies only JSON data from a plain object. Reflection happens before any - * value read, so accessors are never invoked. `structuredClone` rejects a - * Proxy, which closes the remaining caller-controlled object membrane. - */ -function snapshotJson(value: unknown, label: string): unknown { - const copy = (candidate: unknown, path: string): unknown => { - if (candidate === null || typeof candidate === 'string' || typeof candidate === 'boolean') return candidate; - if (typeof candidate === 'number') { - if (!Number.isFinite(candidate)) throw new Error(`${path} contains a non-finite number`); - return candidate; - } - if (typeof candidate !== 'object') throw new Error(`${path} is not JSON data`); - if (Array.isArray(candidate)) { - const descriptors = Object.getOwnPropertyDescriptors(candidate); - if (Object.getPrototypeOf(candidate) !== Array.prototype || Object.getOwnPropertySymbols(candidate).length > 0) { - throw new Error(`${path} must be a plain array without symbols`); - } - for (const [key, descriptor] of Object.entries(descriptors)) { - if (key !== 'length' && (!('value' in descriptor) || !descriptor.enumerable)) { - throw new Error(`${path} contains an accessor or hidden property`); - } - } - return candidate.map((item, index) => copy(item, `${path}[${index}]`)); - } - if (Object.getPrototypeOf(candidate) !== Object.prototype || Object.getOwnPropertySymbols(candidate).length > 0) { - throw new Error(`${path} must be a plain object without symbols`); - } - const descriptors = Object.getOwnPropertyDescriptors(candidate); - const output: Record = {}; - for (const [key, descriptor] of Object.entries(descriptors)) { - if (!('value' in descriptor) || !descriptor.enumerable) { - throw new Error(`${path}.${key} must be an own enumerable data property`); - } - output[key] = copy(descriptor.value, `${path}.${key}`); - } - return output; - }; - // Do this after descriptor validation: structuredClone otherwise invokes a - // getter. It reliably rejects Proxy values that can impersonate descriptors. - try { - structuredClone(value); - } catch { - throw new Error(`${label} must not contain a Proxy or non-cloneable value`); - } - return deepFreeze(copy(value, label)); -} - function assertExactKeys(value: object, keys: readonly string[], label: string): void { const actual = Object.keys(value).sort(); const expected = [...keys].sort(); @@ -365,12 +322,6 @@ function sha256(value: unknown): string { return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); } -function deepFreeze(value: T): T { - if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; - for (const nested of Object.values(value)) deepFreeze(nested); - return Object.freeze(value); -} - function requireHash(value: string, label: string): void { if (!/^[a-f0-9]{64}$/.test(value)) throw new Error(`${label} must be a SHA-256 hex digest`); } @@ -541,13 +492,19 @@ function assertFixedTraceExperimentPlanStructure( } } +/** The sole boundary at which caller data becomes immutable plan data. */ +function validatedPlanSnapshot(plan: FixedTraceExperimentPlan): FixedTraceExperimentPlan { + const snapshot = snapshotFixedTraceJson(plan, 'experiment plan') as FixedTraceExperimentPlan; + assertFixedTraceExperimentPlanStructure(snapshot); + return snapshot; +} + /** Validate an untrusted plan without credentials, providers, outputs, or a resolver. */ export function validateFixedTraceExperimentPlanOffline(plan: FixedTraceExperimentPlan): FixedTraceOfflinePlanValidation { - const snapshot = snapshotJson(plan, 'experiment plan') as FixedTraceExperimentPlan; + const snapshot = validatedPlanSnapshot(plan); // A submitted plan can describe only priced, already reviewed stages. Terra // and Sol have no reviewed repository price, so their descriptors cannot // enter an estimate or a budget reservation. - assertFixedTraceExperimentPlanStructure(snapshot); return Object.freeze({ diagnosticOnly: true, comparisonEligible: false, @@ -566,12 +523,17 @@ export function validateFixedTraceExperimentPlanOffline(plan: FixedTraceExperime export function validateFixedTraceRawAuditableLedgerOffline( plan: FixedTraceExperimentPlan, ledger: FixedTraceRawAuditableLedger, + expectedTrustedManifestSha256: string, ): void { - const safePlan = snapshotJson(plan, 'experiment plan') as FixedTraceExperimentPlan; - const safeLedger = snapshotJson(ledger, 'raw ledger') as FixedTraceRawAuditableLedger; - assertFixedTraceExperimentPlanStructure(safePlan); + const safePlan = validatedPlanSnapshot(plan); + const safeLedger = snapshotFixedTraceJson(ledger, 'raw ledger') as FixedTraceRawAuditableLedger; + requireHash(expectedTrustedManifestSha256, 'expected trustedManifestSha256'); assertExactKeys(safeLedger, ['version', 'trustedManifestSha256', 'planFingerprint', 'budgetIdentitySha256', 'entries'], 'raw ledger'); if (safeLedger.version !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Unsupported raw fixed-trace ledger version'); + if (safeLedger.trustedManifestSha256 !== expectedTrustedManifestSha256) throw new Error('Raw ledger trusted manifest mismatch'); + if (safeLedger.planFingerprint !== sha256(safePlan)) throw new Error('Raw ledger plan fingerprint mismatch'); + const expectedBudgetIdentity = estimateFixedTraceExperiment(safePlan, (() => null) as FixedTraceTrustedManifestResolver).budgetIdentitySha256; + if (safeLedger.budgetIdentitySha256 !== expectedBudgetIdentity) throw new Error('Raw ledger budget identity mismatch'); if (!Array.isArray(safeLedger.entries)) throw new Error('Raw ledger entries must be an array'); const traceById = new Map(FIXED_TRACE_SUITE.map((trace) => [trace.id, trace])); const expected = safePlan.arms.flatMap((arm) => selectedTraceIds(safePlan).flatMap((traceId) => { @@ -584,7 +546,7 @@ export function validateFixedTraceRawAuditableLedgerOffline( if (safeLedger.entries.length !== expected.length) throw new Error('Raw ledger lacks complete planned-stage coverage'); for (const [index, entry] of safeLedger.entries.entries()) { assertExactKeys(entry, [ - 'sequence', 'armId', 'repetitionIndex', 'traceId', 'stage', 'dispatched', + 'sequence', 'phaseId', 'armId', 'repetitionIndex', 'traceId', 'stage', 'callIndex', 'dispatched', 'requestedProvider', 'requestedModel', 'returnedProvider', 'returnedModel', 'promptSha256', 'providerRequestSha256', 'responseSha256', 'rawRequestArtifact', 'rawResponseArtifact', 'exactToolNames', 'caseControlSha256', 'executionEnvelopeSha256', @@ -593,7 +555,7 @@ export function validateFixedTraceRawAuditableLedgerOffline( 'finishReason', 'usage', 'estimatedCostUsd', ], `raw ledger entry ${index + 1}`); const want = expected[index]!; - if (entry.sequence !== index + 1 || entry.armId !== want.arm.id || entry.repetitionIndex !== want.arm.repetitionIndex || entry.traceId !== want.traceId || entry.stage !== want.stage) { + if (entry.sequence !== index + 1 || entry.phaseId !== want.arm.screeningStage || entry.armId !== want.arm.id || entry.repetitionIndex !== want.arm.repetitionIndex || entry.traceId !== want.traceId || entry.stage !== want.stage || entry.callIndex !== 1) { throw new Error('Raw ledger sequence does not exactly match the planned stages'); } const trace = traceById.get(entry.traceId); @@ -659,9 +621,9 @@ export function fixedTraceExperimentRunnerBinding( addieCodeVersion: plan.addieCodeVersion, stageControlVersion: plan.stageControlVersion, promptConfigVersion: plan.promptConfigVersion, - traceSuite: deepFreeze(structuredClone(suite.traceSuite)), + traceSuite: deepFreezeFixedTrace(snapshotFixedTraceJson(suite.traceSuite, 'trusted manifest trace suite')) as ReadonlyArray, traceSuiteSha256: suite.traceSuiteSha256, - toolDefinitions: deepFreeze(structuredClone(suite.toolDefinitions)), + toolDefinitions: deepFreezeFixedTrace(snapshotFixedTraceJson(suite.toolDefinitions, 'trusted manifest tool definitions')) as ReadonlyArray, toolDefinitionProvenance: suite.toolDefinitionProvenance, providerDegradationInjectionEnabled: plan.providerDegradationInjectionEnabled, }); @@ -669,7 +631,8 @@ export function fixedTraceExperimentRunnerBinding( /** Omits only execution partition/finalization state so an approved candidate cannot drift at unlock. */ export function fixedTraceCandidatePlanFingerprint(plan: FixedTraceExperimentPlan): string { - const { partition, ...candidatePlan } = plan; + const snapshot = validatedPlanSnapshot(plan); + const { partition, ...candidatePlan } = snapshot; return sha256({ ...candidatePlan, partition: { @@ -704,8 +667,8 @@ export function assertFixedTraceExperimentPlan( resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, ): void { - const snapshot = snapshotJson(plan, 'experiment plan') as FixedTraceExperimentPlan; - assertFixedTraceExperimentPlanStructure(snapshot, holdoutFinalizationResolver); + const snapshot = validatedPlanSnapshot(plan); + if (snapshot.partition.selected === 'holdout') assertHoldoutFinalization(snapshot, holdoutFinalizationResolver); resolveTrustedManifest(snapshot, resolver); } @@ -719,10 +682,10 @@ export function fixedTraceExperimentPlanFingerprint(plan: FixedTraceExperimentPl export function fixedTraceExperimentExecutionOrder(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): readonly string[] { void resolver; void holdoutFinalizationResolver; - validateFixedTraceExperimentPlanOffline(plan); - return Object.freeze([...plan.arms] - .sort((left, right) => sha256({ seed: plan.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) - .localeCompare(sha256({ seed: plan.ordering.seed, arm: right.id, repetition: right.repetitionIndex })) || left.id.localeCompare(right.id)) + const snapshot = validatedPlanSnapshot(plan); + return Object.freeze([...snapshot.arms] + .sort((left, right) => sha256({ seed: snapshot.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) + .localeCompare(sha256({ seed: snapshot.ordering.seed, arm: right.id, repetition: right.repetitionIndex })) || left.id.localeCompare(right.id)) .map((arm) => arm.id)); } @@ -750,20 +713,20 @@ function reservation( export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): FixedTraceDryRunEstimate { void resolver; void holdoutFinalizationResolver; - validateFixedTraceExperimentPlanOffline(plan); + const snapshot = validatedPlanSnapshot(plan); const candidate: FixedTraceStageReservation[] = []; const judges: FixedTraceStageReservation[] = []; - const traceIds = selectedTraceIds(plan); - for (const arm of plan.arms) { - if (arm.router) candidate.push(reservation(arm, 'router', arm.router, traceIds, plan.pricingAsOf)); - if (arm.generation) candidate.push(reservation(arm, 'generation', arm.generation, traceIds, plan.pricingAsOf)); - for (const judge of arm.judges ?? []) judges.push(reservation(arm, 'judge', judge, traceIds, plan.pricingAsOf)); + const traceIds = selectedTraceIds(snapshot); + for (const arm of snapshot.arms) { + if (arm.router) candidate.push(reservation(arm, 'router', arm.router, traceIds, snapshot.pricingAsOf)); + if (arm.generation) candidate.push(reservation(arm, 'generation', arm.generation, traceIds, snapshot.pricingAsOf)); + for (const judge of arm.judges ?? []) judges.push(reservation(arm, 'judge', judge, traceIds, snapshot.pricingAsOf)); } const candidateCeilingUsd = candidate.reduce((total, item) => total + item.ceilingUsd, 0); const judgeCeilingUsd = judges.reduce((total, item) => total + item.ceilingUsd, 0); - if (candidateCeilingUsd > plan.budgets.candidateCeilingUsd) throw new Error('Candidate worst-case reservation exceeds its separate budget'); - if (judgeCeilingUsd > plan.budgets.judgeCeilingUsd) throw new Error('Judge worst-case reservation exceeds its separate budget'); - const planFingerprint = fixedTraceExperimentPlanFingerprint(plan, resolver, holdoutFinalizationResolver); + if (candidateCeilingUsd > snapshot.budgets.candidateCeilingUsd) throw new Error('Candidate worst-case reservation exceeds its separate budget'); + if (judgeCeilingUsd > snapshot.budgets.judgeCeilingUsd) throw new Error('Judge worst-case reservation exceeds its separate budget'); + const planFingerprint = sha256(snapshot); const budgetIdentitySha256 = sha256({ planFingerprint, candidate: candidate.map((item) => ({ ...item })), @@ -774,7 +737,10 @@ export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, res budgetIdentitySha256, diagnosticOnly: true, comparisonEligible: false, - executionOrder: fixedTraceExperimentExecutionOrder(plan, resolver, holdoutFinalizationResolver), + executionOrder: Object.freeze([...snapshot.arms] + .sort((left, right) => sha256({ seed: snapshot.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) + .localeCompare(sha256({ seed: snapshot.ordering.seed, arm: right.id, repetition: right.repetitionIndex })) || left.id.localeCompare(right.id)) + .map((arm) => arm.id)), candidate: Object.freeze({ ceilingUsd: candidateCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(candidate) }), judges: Object.freeze({ ceilingUsd: judgeCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(judges) }), totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd, diff --git a/server/src/addie/eval/fixed-trace-safe-snapshot.ts b/server/src/addie/eval/fixed-trace-safe-snapshot.ts new file mode 100644 index 0000000000..06149124ee --- /dev/null +++ b/server/src/addie/eval/fixed-trace-safe-snapshot.ts @@ -0,0 +1,69 @@ +import { types } from 'node:util'; + +/** + * Detach hostile JSON-shaped input without ever reading a value through the + * object. `structuredClone` is intentionally not used here: it invokes + * getters before it rejects them. Node exposes proxy identity without + * invoking user traps, which lets this boundary fail before reflection. + */ +export function snapshotFixedTraceJson(value: unknown, label: string): unknown { + const copy = (candidate: unknown, path: string): unknown => { + if (candidate === null || typeof candidate === 'string' || typeof candidate === 'boolean') return candidate; + if (typeof candidate === 'number') { + if (!Number.isFinite(candidate)) throw new Error(`${path} contains a non-finite number`); + return candidate; + } + if (typeof candidate !== 'object') throw new Error(`${path} is not JSON data`); + if (types.isProxy(candidate)) throw new Error(`${path} must not contain a Proxy`); + + if (Array.isArray(candidate)) { + if (Object.getPrototypeOf(candidate) !== Array.prototype || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain array without symbols`); + } + const descriptors = Object.getOwnPropertyDescriptors(candidate) as Record; + const lengthDescriptor = descriptors['length']; + if (!lengthDescriptor || !('value' in lengthDescriptor) || lengthDescriptor.enumerable || !Number.isSafeInteger(lengthDescriptor.value) || lengthDescriptor.value < 0) { + throw new Error(`${path} has an invalid array length descriptor`); + } + const length = lengthDescriptor.value as number; + const output: unknown[] = []; + for (const key of Object.keys(descriptors)) { + if (key === 'length') continue; + if (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= length) { + throw new Error(`${path} contains an extra array property`); + } + } + for (let index = 0; index < length; index += 1) { + const descriptor = descriptors[String(index)]; + if (!descriptor || !('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}[${index}] must be an own enumerable data property`); + } + output.push(copy(descriptor.value, `${path}[${index}]`)); + } + return output; + } + + if (Object.getPrototypeOf(candidate) !== Object.prototype || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain object without symbols`); + } + const descriptors = Object.getOwnPropertyDescriptors(candidate); + const output: Record = {}; + for (const [key, descriptor] of Object.entries(descriptors)) { + if (!('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}.${key} must be an own enumerable data property`); + } + output[key] = copy(descriptor.value, `${path}.${key}`); + } + return output; + }; + return deepFreezeFixedTrace(copy(value, label)); +} + +/** Freeze only detached JSON data, never an object supplied by a caller. */ +export function deepFreezeFixedTrace(value: T): T { + if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; + for (const descriptor of Object.values(Object.getOwnPropertyDescriptors(value))) { + if ('value' in descriptor) deepFreezeFixedTrace(descriptor.value); + } + return Object.freeze(value); +} diff --git a/server/src/training-agent/reporting-reliability.ts b/server/src/training-agent/reporting-reliability.ts index c4a306011a..af403d3364 100644 --- a/server/src/training-agent/reporting-reliability.ts +++ b/server/src/training-agent/reporting-reliability.ts @@ -1228,7 +1228,7 @@ export function publishReportingCoreLifecycleProbeRows( }, rows); ledger.publishedRevisions.set(obligation, revision); ledger.version += 1; - return { reporting_revision_id: revision.reporting_revision_id, row_count: revision.row_count, revision_content_sha256: revision.revision_content_sha256 }; + return { reporting_revision_id: revision.reporting_revision_id, row_count: revision.row_count, revision_content_sha256: (revision as unknown as { revision_content_sha256: string }).revision_content_sha256 }; } /** @@ -2268,7 +2268,7 @@ function commitRevisionContent( reporting_rows: rows, })).digest('hex'); const existing = ledger.revisionContents.get(revision.reporting_revision_id); - const committed: ReportingRevision = { ...structuredClone(revision), revision_content_sha256: bindingSha256 }; + const committed = { ...structuredClone(revision), revision_content_sha256: bindingSha256 } as ReportingRevision; if (existing) { // Identity binds the complete metadata, authoritative rows, and binding // digest. Exact retries return the originally committed revision bytes. diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index 11f544366f..d586fcf78c 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -326,7 +326,7 @@ if (cliArguments.validateOnly) { process.exit(0); } if (!outputArgument?.trim()) throw new Error('--output is required'); -const outputPath = resolve(outputArgument); +resolve(outputArgument); throw new Error('Live fixed-trace replay is disabled pending an evaluator-owned execution-contract review'); // This exclusive create happens before source inspection, credentials, // provider construction, or dispatch. Never unlink it: an empty file is the diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index e9a434a5a4..3e11f497de 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -1,11 +1,24 @@ import { describe, expect, it } from 'vitest'; -import { FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; +import { createHash } from 'node:crypto'; +import { fixedTraceEstimatedCostUsd } from '../../../src/addie/eval/fixed-trace-budget.js'; +import { FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; + +function historicalOwnEnumerableFingerprint(value: unknown): string { + const canonical = (current: unknown): string => { + if (current === null || typeof current === 'boolean' || typeof current === 'string' || typeof current === 'number') return JSON.stringify(current); + if (Array.isArray(current)) return `[${current.map(canonical).join(',')}]`; + if (typeof current === 'object') return `{${Object.keys(current).sort().map((key) => `${JSON.stringify(key)}:${canonical((current as Record)[key])}`).join(',')}}`; + throw new Error('not JSON'); + }; + return createHash('sha256').update(canonical(value), 'utf8').digest('hex'); +} describe('fixed-trace evaluation protocol projection', () => { it('is ordered, diagnostic-only, non-dispatchable, and non-promotional', () => { const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); assertFixedTraceEvaluationProtocol(protocol); - expect(protocol.phases.map((phase) => phase.id)).toEqual(['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning', 'sealed_final']); + expect(protocol.phases.map((phase) => phase.id)).toEqual(['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning']); + expect(protocol.unavailableFinalTarget).toEqual({ availability: 'unavailable', uniqueCaseCount: 38, repetitions: 3, missingCaseCount: 38 }); expect(protocol.phases.every((phase) => phase.resultUse === 'diagnostic_only')).toBe(true); expect(estimateFixedTraceEvaluationProtocol(protocol)).toMatchObject({ dispatchable: false, expectedSpendUsd: null }); }); @@ -25,4 +38,45 @@ describe('fixed-trace evaluation protocol projection', () => { expect(() => assertFixedTraceEvaluationProtocolTrusted(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any)).toThrow('locked'); expect(() => fixedTraceEvaluationProtocolRunnerBinding(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any, 'bounded_smoke', [])).toThrow('locked'); }); + + it('uses a detached closed snapshot for validation, hashing, and estimates', () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + const expectedFingerprint = fixedTraceEvaluationProtocolFingerprint(protocol); + const estimate = estimateFixedTraceEvaluationProtocol(protocol); + protocol.phases[1].arms[0].stages[0].maxOutputTokensPerInvocation = 999; + expect(estimate.stages.find((stage) => stage.phaseId === 'router_screen')?.outputTokenCeiling).toBe(46 * 3 * 300); + expect(Object.isFrozen(estimate)).toBe(true); + expect(Object.isFrozen(estimate.phases)).toBe(true); + expect(expectedFingerprint).not.toBe(fixedTraceEvaluationProtocolFingerprint(protocol)); + + const arrayExtra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + arrayExtra.phases.extra = true; + expect(() => assertFixedTraceEvaluationProtocol(arrayExtra)).toThrow('extra array property'); + let getterReads = 0; + const accessor = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + Object.defineProperty(accessor, 'id', { enumerable: true, get() { getterReads += 1; return 'forged'; } }); + expect(() => assertFixedTraceEvaluationProtocol(accessor)).toThrow('own enumerable data'); + expect(getterReads).toBe(0); + expect(() => assertFixedTraceEvaluationProtocol(new Proxy(protocol, {}))).toThrow('Proxy'); + }); + + it('rejects inherited Anthropic-to-Google stage substitution before it can alter cost or a fingerprint', () => { + const inheritedProtocol = (provider: 'anthropic' | 'google', model: string, pricingProfileId: string) => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + const stage = protocol.phases[1].arms[0].stages[0]; + const { provider: ignoredProvider, model: ignoredModel, pricingProfileId: ignoredPricing, ...ownFields } = stage; + void ignoredProvider; void ignoredModel; void ignoredPricing; + protocol.phases[1].arms[0].stages[0] = Object.assign(Object.create({ provider, model, pricingProfileId }), ownFields); + return protocol; + }; + const anthropic = FIXED_TRACE_PROTOCOL_PRICING.find((profile) => profile.provider === 'anthropic' && profile.model === 'claude-haiku-4-5')!; + const google = FIXED_TRACE_PROTOCOL_PRICING.find((profile) => profile.provider === 'google')!; + const inheritedAnthropic = inheritedProtocol('anthropic', anthropic.model, anthropic.profileId); + const inheritedGoogle = inheritedProtocol('google', google.model, google.profileId); + expect(historicalOwnEnumerableFingerprint(inheritedGoogle)).toBe(historicalOwnEnumerableFingerprint(inheritedAnthropic)); + expect(fixedTraceEstimatedCostUsd({ inputTokens: 46 * 3 * 4_096, outputTokens: 46 * 3 * 300, cacheReadTokens: 0, cacheWriteTokens: 0 }, google)) + .not.toBe(fixedTraceEstimatedCostUsd({ inputTokens: 46 * 3 * 4_096, outputTokens: 46 * 3 * 300, cacheReadTokens: 0, cacheWriteTokens: 0 }, anthropic)); + expect(() => fixedTraceEvaluationProtocolFingerprint(inheritedAnthropic)).toThrow('plain object'); + expect(() => fixedTraceEvaluationProtocolFingerprint(inheritedGoogle)).toThrow('plain object'); + }); }); diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts index 46f6ba4fcd..c1e0aa5a38 100644 --- a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts +++ b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; +import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, estimateFixedTraceExperiment, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; import { FIXED_TRACE_PARTITION_MANIFEST, FIXED_TRACE_PARTITION_MANIFEST_SHA256, FIXED_TRACE_PARTITION_MANIFEST_VERSION } from '../../../src/addie/eval/fixed-trace-partition.js'; import { CLAUDE_PRICING_VERSION } from '../../../src/addie/claude-pricing.js'; import { CODE_VERSION } from '../../../src/addie/config-version.js'; @@ -26,16 +26,33 @@ describe('fixed-trace experiment plan offline boundary', () => { const terra = plan(); terra.arms[0].router!.model = 'gpt-5.6-terra'; expect(() => validateFixedTraceExperimentPlanOffline(terra)).toThrow('Unavailable immutable pricing'); }); + it('does not invoke a hostile getter before rejecting it, and detaches estimates', () => { + const hostile = plan() as any; + let reads = 0; + Object.defineProperty(hostile, 'id', { enumerable: true, get() { reads += 1; return 'forged'; } }); + expect(() => validateFixedTraceExperimentPlanOffline(hostile)).toThrow('own enumerable data'); + expect(reads).toBe(0); + const mutable = plan(); + const estimate = estimateFixedTraceExperiment(mutable, () => null); + mutable.arms[0].router!.maxOutputTokens = 999; + expect(estimate.candidate.reservations[0]?.outputTokens).toBe(FIXED_TRACE_PARTITION_MANIFEST.development.length * 10); + expect(Object.isFrozen(estimate.candidate.reservations)).toBe(true); + (mutable.arms as any).extra = true; + expect(() => validateFixedTraceExperimentPlanOffline(mutable)).toThrow('extra array property'); + }); it('requires exact ledger sequence, tools, and offline provider resolution', () => { const current = plan(); - const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); - const ledger = { version: 'addie-fixed-trace-raw-ledger-v1' as const, trustedManifestSha256: HASH, planFingerprint: HASH, budgetIdentitySha256: HASH, entries }; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).not.toThrow(); + const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, phaseId: 'router_only_screen' as const, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, callIndex: 1 as const, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); + const ledger = { version: 'addie-fixed-trace-raw-ledger-v1' as const, trustedManifestSha256: HASH, planFingerprint: validateFixedTraceExperimentPlanOffline(current).planFingerprint, budgetIdentitySha256: estimateFixedTraceExperiment(current, () => null).budgetIdentitySha256, entries }; + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).not.toThrow(); ledger.entries[1].sequence = 1; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).toThrow('sequence'); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('sequence'); ledger.entries[1].sequence = 2; ledger.entries[0].exactToolNames = ['tampered']; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).toThrow('tool names'); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('tool names'); ledger.entries[0].exactToolNames = FIXED_TRACE_SUITE.find((item) => item.id === ledger.entries[0].traceId)!.toolFixtures.map((fixture) => fixture.name); ledger.entries[0].returnedProvider = 'google'; ledger.entries[0].returnedModel = 'gemini-3.7-flash'; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger)).toThrow('dispatch, response'); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('dispatch, response'); + ledger.entries[0].returnedProvider = null; ledger.entries[0].returnedModel = null; + ledger.trustedManifestSha256 = 'b'.repeat(64); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('trusted manifest mismatch'); }); }); From 425bcc593af45c39bb8a8481d16ca0a099683e5b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 17:50:55 +0000 Subject: [PATCH 07/16] chore(addie): restore main reporting typing --- server/src/training-agent/reporting-reliability.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/training-agent/reporting-reliability.ts b/server/src/training-agent/reporting-reliability.ts index af403d3364..c4a306011a 100644 --- a/server/src/training-agent/reporting-reliability.ts +++ b/server/src/training-agent/reporting-reliability.ts @@ -1228,7 +1228,7 @@ export function publishReportingCoreLifecycleProbeRows( }, rows); ledger.publishedRevisions.set(obligation, revision); ledger.version += 1; - return { reporting_revision_id: revision.reporting_revision_id, row_count: revision.row_count, revision_content_sha256: (revision as unknown as { revision_content_sha256: string }).revision_content_sha256 }; + return { reporting_revision_id: revision.reporting_revision_id, row_count: revision.row_count, revision_content_sha256: revision.revision_content_sha256 }; } /** @@ -2268,7 +2268,7 @@ function commitRevisionContent( reporting_rows: rows, })).digest('hex'); const existing = ledger.revisionContents.get(revision.reporting_revision_id); - const committed = { ...structuredClone(revision), revision_content_sha256: bindingSha256 } as ReportingRevision; + const committed: ReportingRevision = { ...structuredClone(revision), revision_content_sha256: bindingSha256 }; if (existing) { // Identity binds the complete metadata, authoritative rows, and binding // digest. Exact retries return the originally committed revision bytes. From 01499233c172f082b9ab7daa8e2bd515610289c3 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 18:05:17 +0000 Subject: [PATCH 08/16] fix(addie): lock evaluation estimate matrix --- .../eval/fixed-trace-evaluation-protocol.ts | 94 +++++++++++++++++-- .../fixed-trace-evaluation-protocol.test.ts | 52 ++++++++++ 2 files changed, 138 insertions(+), 8 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index b6f7fff13b..ce869cae8c 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -292,6 +292,85 @@ function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: return resolved; } +/** + * Execution limits are evaluator-owned planning inputs, not caller-selected + * estimates. Keep this matrix independent of the proposed protocol object so + * a detached protocol supplied to an offline estimator cannot rewrite its + * phase, admission, result-use, or stop conditions. + */ +const EVALUATOR_OWNED_PHASE_MATRIX = Object.freeze([ + Object.freeze({ + id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, + arms: Object.freeze([Object.freeze({ + id: 'smoke-incumbent-two-stage', admission: 'planning_only', + stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + })]), + }), + Object.freeze({ + id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, + arms: Object.freeze([Object.freeze({ + id: 'router-haiku-default', admission: 'planning_only', + stopConditions: Object.freeze([['router', 1]] as const), + })]), + }), + Object.freeze({ + id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, + arms: Object.freeze([Object.freeze({ + id: 'oracle-sonnet-default', admission: 'planning_only', + stopConditions: Object.freeze([['generation', 12]] as const), + })]), + }), + Object.freeze({ + id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, + arms: Object.freeze([ + Object.freeze({ + id: 'incumbent-haiku-sonnet', admission: 'planning_only', + stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + }), + Object.freeze({ + id: 'gemini-low-medium-pipeline', admission: 'planning_only', + stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + }), + ]), + }), + Object.freeze({ + id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, + arms: Object.freeze([Object.freeze({ + id: 'tuning-incumbent-haiku-sonnet', admission: 'planning_only', + stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + })]), + }), +] as const); + +function assertEvaluatorOwnedPhaseMatrix(phase: FixedTraceProtocolPhase, index: number): void { + const expected = EVALUATOR_OWNED_PHASE_MATRIX[index]; + if (!expected + || phase.id !== expected.id + || phase.uniqueCaseCount !== expected.uniqueCaseCount + || phase.repetitions !== expected.repetitions + || phase.resultUse !== 'diagnostic_only') { + throw new Error('Protocol phase does not match the evaluator-owned phase matrix'); + } + if (phase.arms.length !== expected.arms.length) { + throw new Error(`${phase.id} arms do not match the evaluator-owned phase matrix`); + } + for (let armIndex = 0; armIndex < phase.arms.length; armIndex += 1) { + const arm = phase.arms[armIndex]; + const expectedArm = expected.arms[armIndex]; + if (!expectedArm || arm.id !== expectedArm.id || arm.admission !== expectedArm.admission) { + throw new Error(`${phase.id} arm does not match the evaluator-owned admission matrix`); + } + if (arm.stages.length !== expectedArm.stopConditions.length || arm.stages.some((stage, stageIndex) => { + const expectedStop = expectedArm.stopConditions[stageIndex]; + return !expectedStop + || stage.role !== expectedStop[0] + || stage.maxInvocationsPerCase !== expectedStop[1]; + })) { + throw new Error(`${phase.id}.${arm.id} does not match the evaluator-owned stop-condition matrix`); + } + } +} + function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, pricingAsOf: string): void { assertExactKeys(arm, ['id', 'architecture', 'admission', 'stages'], `protocol arm`); if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid protocol arm ID: ${arm.id}`); @@ -350,25 +429,24 @@ function assertFixedTraceEvaluationProtocolStructure(protocol: FixedTraceEvaluat || protocol.unavailableFinalTarget.missingCaseCount !== 38) { throw new Error('Protocol unavailable final target is invalid'); } - const requiredOrder = ['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning'] as const; - if (protocol.phases.length !== requiredOrder.length || protocol.phases.some((phase, index) => phase.id !== requiredOrder[index])) { + if (protocol.phases.length !== EVALUATOR_OWNED_PHASE_MATRIX.length + || protocol.phases.some((phase, index) => phase.id !== EVALUATOR_OWNED_PHASE_MATRIX[index]?.id)) { throw new Error('Protocol phases must use the exact required order'); } - for (const phase of protocol.phases) { + for (const [index, phase] of protocol.phases.entries()) { assertExactKeys(phase, ['id', 'uniqueCaseCount', 'repetitions', 'resultUse', 'arms'], 'protocol phase'); if (phaseIds.has(phase.id)) throw new Error(`Duplicate protocol phase: ${phase.id}`); phaseIds.add(phase.id); - positiveInteger(phase.uniqueCaseCount, `${phase.id}.uniqueCaseCount`); - positiveInteger(phase.repetitions, `${phase.id}.repetitions`); - if (phase.resultUse !== 'diagnostic_only') throw new Error(`${phase.id} is not diagnostic-only`); - if (!phase.arms.length) throw new Error(`${phase.id} requires at least one arm`); + assertEvaluatorOwnedPhaseMatrix(phase, index); for (const arm of phase.arms) { if (armIds.has(arm.id)) throw new Error(`Duplicate protocol arm ID: ${arm.id}`); armIds.add(arm.id); assertArm(phase, arm, protocol.pricingAsOf); } } - for (const required of requiredOrder) if (!phaseIds.has(required)) throw new Error(`Protocol is missing required phase: ${required}`); + for (const required of EVALUATOR_OWNED_PHASE_MATRIX) { + if (!phaseIds.has(required.id)) throw new Error(`Protocol is missing required phase: ${required.id}`); + } } export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): void { diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index 3e11f497de..07fe23915f 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -39,6 +39,58 @@ describe('fixed-trace evaluation protocol projection', () => { expect(() => fixedTraceEvaluationProtocolRunnerBinding(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any, 'bounded_smoke', [])).toThrow('locked'); }); + it('rejects the reported caller substitutions before estimating a budget', () => { + const substituted = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + const phase = substituted.phases[0]; + phase.arms[0].admission = 'caller_promotional'; + phase.uniqueCaseCount = 1; + phase.repetitions = 999; + phase.arms[0].stages[0].maxInvocationsPerCase = 999; + expect(() => estimateFixedTraceEvaluationProtocol(substituted)).toThrow('evaluator-owned'); + }); + + it('rejects missing, extra, reordered, and substituted available phases', () => { + const missing = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + missing.phases.splice(2, 1); + expect(() => estimateFixedTraceEvaluationProtocol(missing)).toThrow('exact required order'); + + const extra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + extra.phases.push(structuredClone(extra.phases[0])); + expect(() => estimateFixedTraceEvaluationProtocol(extra)).toThrow('exact required order'); + + const reordered = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + [reordered.phases[0], reordered.phases[1]] = [reordered.phases[1], reordered.phases[0]]; + expect(() => estimateFixedTraceEvaluationProtocol(reordered)).toThrow('exact required order'); + + const substituted = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + substituted.phases[3].arms[1] = structuredClone(substituted.phases[3].arms[0]); + expect(() => estimateFixedTraceEvaluationProtocol(substituted)).toThrow('evaluator-owned admission matrix'); + }); + + it('enforces evaluator-owned admission, result use, counts, repetitions, and stop conditions for every phase', () => { + for (let phaseIndex = 0; phaseIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.length; phaseIndex += 1) { + for (const mutate of [ + (phase: any) => { phase.uniqueCaseCount = 1; }, + (phase: any) => { phase.repetitions = 999; }, + (phase: any) => { phase.resultUse = 'caller_promotional'; }, + ]) { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + mutate(protocol.phases[phaseIndex]); + expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow('evaluator-owned phase matrix'); + } + for (let armIndex = 0; armIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms.length; armIndex += 1) { + const admission = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + admission.phases[phaseIndex].arms[armIndex].admission = 'caller_promotional'; + expect(() => estimateFixedTraceEvaluationProtocol(admission)).toThrow('evaluator-owned admission matrix'); + for (let stageIndex = 0; stageIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms[armIndex].stages.length; stageIndex += 1) { + const stopCondition = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + stopCondition.phases[phaseIndex].arms[armIndex].stages[stageIndex].maxInvocationsPerCase = 999; + expect(() => estimateFixedTraceEvaluationProtocol(stopCondition)).toThrow('evaluator-owned stop-condition matrix'); + } + } + } + }); + it('uses a detached closed snapshot for validation, hashing, and estimates', () => { const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); const expectedFingerprint = fixedTraceEvaluationProtocolFingerprint(protocol); From 3e1ff13febde7c5d06f1cb5d7ce5a80044293e6d Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 18:14:06 +0000 Subject: [PATCH 09/16] fix(addie): gate confirmatory evaluation power --- .../eval/fixed-trace-evaluation-protocol.ts | 113 ++++++++++++++++++ .../fixed-trace-evaluation-protocol.test.ts | 41 ++++++- 2 files changed, 152 insertions(+), 2 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index ce869cae8c..5c4adcad7d 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -21,6 +21,34 @@ import { deepFreezeFixedTrace, snapshotFixedTraceJson } from './fixed-trace-safe export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = 'addie-fixed-trace-evaluation-protocol-v1' as const; +/** + * Evaluator-owned confirmatory precision rule. The conservative normal + * approximation assumes the maximum possible variance (1) of a paired + * case-level difference in [-1, 1], with one-sided alpha .025 and 80% power. + * The non-inferiority margin is limiting: ceil((1.9599639845 + .8416212336)^2 + * / .03^2) = 8,721 independent paired cases. This is deliberately a sample + * requirement, not a price quote or an authorization to spend. + */ +export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ + version: 'addie-fixed-trace-confirmatory-power-v1', + unit: 'unique_paired_case', + repetitionsCountAsIndependentCases: false, + oneSidedAlpha: 0.025, + targetPower: 0.8, + conservativePairedDifferenceVarianceUpperBound: 1, + superiorityMarginPercentagePoints: 5, + nonInferiorityMarginPercentagePoints: -3, + superiorityRequiredIndependentEvaluableCases: 3_140, + nonInferiorityRequiredIndependentEvaluableCases: 8_721, + requiredIndependentEvaluableCases: 8_721, + requiredAnalysis: Object.freeze({ + resampling: 'grouped_stratified_case_level_bootstrap', + multiplicityCorrection: 'holm', + pairedDiscordanceTest: 'predeclared_exact_paired_test_required', + }), + currentScreeningTuningUniqueCaseCount: 120, +} as const); + export type FixedTraceProtocolPhaseId = | 'bounded_smoke' | 'router_screen' @@ -223,12 +251,82 @@ export interface FixedTraceProtocolEstimate { phases: readonly FixedTraceProtocolPhaseEstimate[]; screening: { candidateCeilingUsd: number; judgeCeilingUsd: number; totalCeilingUsd: number }; unavailableFinalTarget: FixedTraceEvaluationProtocol['unavailableFinalTarget']; + /** The confirmatory sample remains unpriced and cannot authorize spend. */ + budgetProjection: { + screeningTuning: { + uniqueEvaluableCaseCount: number; + repetitionsCountAsIndependentCases: false; + expectedSpendUsd: null; + approvalCeilingUsd: null; + }; + confirmatory: { + requiredIndependentEvaluableCaseCount: number; + unavailableTargetCaseCount: number; + expectedSpendUsd: null; + approvalCeilingUsd: null; + spendAuthorization: 'refused_pending_evaluator_owned_paired_test'; + }; + }; candidateCeilingUsd: number; judgeCeilingUsd: number; contingencyUsd: number; totalCeilingUsd: number; } +export interface FixedTraceConfirmatoryClaimInput { + /** One entry per observed paired evaluation; repeated IDs remain one case. */ + pairedCaseIds: readonly string[]; + observedSuperiorityPercentagePoints: number; + observedNonInferiorityPercentagePoints: number; +} + +export interface FixedTraceConfirmatoryClaimGate { + independentEvaluableCaseCount: number; + repeatedObservationCount: number; + requiredIndependentEvaluableCaseCount: number; + nominalMarginsReached: boolean; + confirmatoryClaim: 'refused_underpowered' | 'refused_pending_evaluator_owned_paired_test'; +} + +/** + * Counts only distinct paired case IDs. Crossing a nominal quality margin is + * descriptive until the evaluator has both the predeclared sample and its + * grouped/stratified case-level bootstrap, Holm correction, and exact paired + * discordance test. This offline planner can never promote a candidate. + */ +export function evaluateFixedTraceConfirmatoryClaim( + input: FixedTraceConfirmatoryClaimInput, +): FixedTraceConfirmatoryClaimGate { + const snapshot = snapshotFixedTraceJson(input, 'confirmatory claim') as FixedTraceConfirmatoryClaimInput; + assertExactKeys(snapshot, [ + 'pairedCaseIds', 'observedSuperiorityPercentagePoints', 'observedNonInferiorityPercentagePoints', + ], 'confirmatory claim'); + const pairedCaseIds = snapshot.pairedCaseIds; + if (pairedCaseIds.some((caseId) => typeof caseId !== 'string' || !caseId.trim())) { + throw new Error('Confirmatory paired case IDs must be nonblank strings'); + } + if (!Number.isFinite(snapshot.observedSuperiorityPercentagePoints) + || !Number.isFinite(snapshot.observedNonInferiorityPercentagePoints)) { + throw new Error('Confirmatory observed margins must be finite'); + } + const independentEvaluableCaseCount = new Set(pairedCaseIds).size; + const repeatedObservationCount = pairedCaseIds.length - independentEvaluableCaseCount; + const nominalMarginsReached = snapshot.observedSuperiorityPercentagePoints + >= FIXED_TRACE_CONFIRMATORY_POWER_GATE.superiorityMarginPercentagePoints + && snapshot.observedNonInferiorityPercentagePoints + >= FIXED_TRACE_CONFIRMATORY_POWER_GATE.nonInferiorityMarginPercentagePoints; + return Object.freeze({ + independentEvaluableCaseCount, + repeatedObservationCount, + requiredIndependentEvaluableCaseCount: FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases, + nominalMarginsReached, + confirmatoryClaim: independentEvaluableCaseCount + < FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases + ? 'refused_underpowered' + : 'refused_pending_evaluator_owned_paired_test', + }); +} + function canonicalJson(value: unknown): string { if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); if (typeof value === 'number') { @@ -557,6 +655,21 @@ export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluat phases: Object.freeze(phases), screening: summarize(phases), unavailableFinalTarget: snapshot.unavailableFinalTarget, + budgetProjection: Object.freeze({ + screeningTuning: Object.freeze({ + uniqueEvaluableCaseCount: FIXED_TRACE_CONFIRMATORY_POWER_GATE.currentScreeningTuningUniqueCaseCount, + repetitionsCountAsIndependentCases: false, + expectedSpendUsd: null, + approvalCeilingUsd: null, + }), + confirmatory: Object.freeze({ + requiredIndependentEvaluableCaseCount: FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases, + unavailableTargetCaseCount: snapshot.unavailableFinalTarget.uniqueCaseCount, + expectedSpendUsd: null, + approvalCeilingUsd: null, + spendAuthorization: 'refused_pending_evaluator_owned_paired_test', + }), + }), candidateCeilingUsd, judgeCeilingUsd, contingencyUsd, diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index 07fe23915f..f8f5c162f2 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { createHash } from 'node:crypto'; import { fixedTraceEstimatedCostUsd } from '../../../src/addie/eval/fixed-trace-budget.js'; -import { FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; +import { FIXED_TRACE_CONFIRMATORY_POWER_GATE, FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, evaluateFixedTraceConfirmatoryClaim, fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; function historicalOwnEnumerableFingerprint(value: unknown): string { const canonical = (current: unknown): string => { @@ -20,7 +20,44 @@ describe('fixed-trace evaluation protocol projection', () => { expect(protocol.phases.map((phase) => phase.id)).toEqual(['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning']); expect(protocol.unavailableFinalTarget).toEqual({ availability: 'unavailable', uniqueCaseCount: 38, repetitions: 3, missingCaseCount: 38 }); expect(protocol.phases.every((phase) => phase.resultUse === 'diagnostic_only')).toBe(true); - expect(estimateFixedTraceEvaluationProtocol(protocol)).toMatchObject({ dispatchable: false, expectedSpendUsd: null }); + expect(estimateFixedTraceEvaluationProtocol(protocol)).toMatchObject({ + dispatchable: false, + expectedSpendUsd: null, + budgetProjection: { + screeningTuning: { uniqueEvaluableCaseCount: 120, approvalCeilingUsd: null }, + confirmatory: { requiredIndependentEvaluableCaseCount: 8_721, unavailableTargetCaseCount: 38, approvalCeilingUsd: null }, + }, + }); + }); + + it('labels nominal 38-case margins inconclusive and does not treat repeated generations as independent cases', () => { + const nominalAt38 = evaluateFixedTraceConfirmatoryClaim({ + pairedCaseIds: Array.from({ length: 38 }, (_, index) => `case-${index + 1}`), + observedSuperiorityPercentagePoints: 5.1, + observedNonInferiorityPercentagePoints: -2.9, + }); + expect(nominalAt38).toMatchObject({ + independentEvaluableCaseCount: 38, + nominalMarginsReached: true, + confirmatoryClaim: 'refused_underpowered', + }); + + const repeatedGenerations = evaluateFixedTraceConfirmatoryClaim({ + pairedCaseIds: Array.from({ length: 38 * 3 }, (_, index) => `case-${index % 38}`), + observedSuperiorityPercentagePoints: 5.1, + observedNonInferiorityPercentagePoints: -2.9, + }); + expect(repeatedGenerations).toMatchObject({ + independentEvaluableCaseCount: 38, + repeatedObservationCount: 76, + requiredIndependentEvaluableCaseCount: 8_721, + confirmatoryClaim: 'refused_underpowered', + }); + expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredAnalysis).toEqual({ + resampling: 'grouped_stratified_case_level_bootstrap', + multiplicityCorrection: 'holm', + pairedDiscordanceTest: 'predeclared_exact_paired_test_required', + }); }); it('keeps Terra and Sol as unpriced inert descriptors', () => { expect(FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES).toEqual([{ provider: 'openai', model: 'gpt-5.6-terra', dispatchable: false, trustedPrice: null }, { provider: 'openai', model: 'gpt-5.6-sol', dispatchable: false, trustedPrice: null }]); From 4439a16071f9ae6ea5217dd5e66d7e91e7efcd64 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 18:24:57 +0000 Subject: [PATCH 10/16] fix(addie): harden fixed trace snapshot membrane --- .../eval/fixed-trace-evaluation-protocol.ts | 3 + .../addie/eval/fixed-trace-experiment-plan.ts | 3 + .../addie/eval/fixed-trace-safe-snapshot.ts | 87 ++++++++++++------- .../fixed-trace-evaluation-protocol.test.ts | 48 ++++++++++ .../addie/fixed-trace-experiment-plan.test.ts | 18 +++- 5 files changed, 125 insertions(+), 34 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index 5c4adcad7d..5104beb651 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -351,6 +351,9 @@ function positiveInteger(value: number, label: string): void { function assertExactKeys(value: object, keys: readonly string[], label: string): void { const actual = Object.keys(value).sort(); + if (actual.some((key) => key === '__proto__' || key === 'prototype' || key === 'constructor')) { + throw new Error(`${label} contains a dangerous prototype key`); + } const expected = [...keys].sort(); if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { throw new Error(`${label} has unknown, missing, or inherited fields`); diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts index 63cdd0c7c6..91908ea507 100644 --- a/server/src/addie/eval/fixed-trace-experiment-plan.ts +++ b/server/src/addie/eval/fixed-trace-experiment-plan.ts @@ -298,6 +298,9 @@ export interface FixedTraceOfflinePlanValidation { function assertExactKeys(value: object, keys: readonly string[], label: string): void { const actual = Object.keys(value).sort(); + if (actual.some((key) => key === '__proto__' || key === 'prototype' || key === 'constructor')) { + throw new Error(`${label} contains a dangerous prototype key`); + } const expected = [...keys].sort(); if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { throw new Error(`${label} has unknown, missing, or inherited fields`); diff --git a/server/src/addie/eval/fixed-trace-safe-snapshot.ts b/server/src/addie/eval/fixed-trace-safe-snapshot.ts index 06149124ee..875c05a955 100644 --- a/server/src/addie/eval/fixed-trace-safe-snapshot.ts +++ b/server/src/addie/eval/fixed-trace-safe-snapshot.ts @@ -7,6 +7,10 @@ import { types } from 'node:util'; * invoking user traps, which lets this boundary fail before reflection. */ export function snapshotFixedTraceJson(value: unknown, label: string): unknown { + // Track only the active ancestry: aliases may be copied as separate JSON + // subtrees, while an actual cycle has no JSON representation and must fail + // before recursion can exhaust the stack. + const activeAncestors = new WeakSet(); const copy = (candidate: unknown, path: string): unknown => { if (candidate === null || typeof candidate === 'string' || typeof candidate === 'boolean') return candidate; if (typeof candidate === 'number') { @@ -15,46 +19,63 @@ export function snapshotFixedTraceJson(value: unknown, label: string): unknown { } if (typeof candidate !== 'object') throw new Error(`${path} is not JSON data`); if (types.isProxy(candidate)) throw new Error(`${path} must not contain a Proxy`); + if (activeAncestors.has(candidate)) throw new Error(`${path} must not contain a cycle`); + activeAncestors.add(candidate); - if (Array.isArray(candidate)) { - if (Object.getPrototypeOf(candidate) !== Array.prototype || Object.getOwnPropertySymbols(candidate).length !== 0) { - throw new Error(`${path} must be a plain array without symbols`); - } - const descriptors = Object.getOwnPropertyDescriptors(candidate) as Record; - const lengthDescriptor = descriptors['length']; - if (!lengthDescriptor || !('value' in lengthDescriptor) || lengthDescriptor.enumerable || !Number.isSafeInteger(lengthDescriptor.value) || lengthDescriptor.value < 0) { - throw new Error(`${path} has an invalid array length descriptor`); - } - const length = lengthDescriptor.value as number; - const output: unknown[] = []; - for (const key of Object.keys(descriptors)) { - if (key === 'length') continue; - if (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= length) { - throw new Error(`${path} contains an extra array property`); + try { + if (Array.isArray(candidate)) { + if (Object.getPrototypeOf(candidate) !== Array.prototype || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain array without symbols`); } - } - for (let index = 0; index < length; index += 1) { - const descriptor = descriptors[String(index)]; - if (!descriptor || !('value' in descriptor) || !descriptor.enumerable) { - throw new Error(`${path}[${index}] must be an own enumerable data property`); + const descriptors = Object.getOwnPropertyDescriptors(candidate) as Record; + const lengthDescriptor = descriptors['length']; + if (!lengthDescriptor || !('value' in lengthDescriptor) || lengthDescriptor.enumerable || !Number.isSafeInteger(lengthDescriptor.value) || lengthDescriptor.value < 0) { + throw new Error(`${path} has an invalid array length descriptor`); } - output.push(copy(descriptor.value, `${path}[${index}]`)); + const length = lengthDescriptor.value as number; + const output: unknown[] = []; + for (const key of Object.keys(descriptors)) { + if (key === 'length') continue; + if (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= length) { + throw new Error(`${path} contains an extra array property`); + } + } + for (let index = 0; index < length; index += 1) { + const descriptor = descriptors[String(index)]; + if (!descriptor || !('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}[${index}] must be an own enumerable data property`); + } + output.push(copy(descriptor.value, `${path}[${index}]`)); + } + return output; } - return output; - } - if (Object.getPrototypeOf(candidate) !== Object.prototype || Object.getOwnPropertySymbols(candidate).length !== 0) { - throw new Error(`${path} must be a plain object without symbols`); - } - const descriptors = Object.getOwnPropertyDescriptors(candidate); - const output: Record = {}; - for (const [key, descriptor] of Object.entries(descriptors)) { - if (!('value' in descriptor) || !descriptor.enumerable) { - throw new Error(`${path}.${key} must be an own enumerable data property`); + // Null-prototype records are this membrane's own detached output and + // are also safe to snapshot again at composed plan/ledger boundaries. + const prototype = Object.getPrototypeOf(candidate); + if ((prototype !== Object.prototype && prototype !== null) || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain object without symbols`); } - output[key] = copy(descriptor.value, `${path}.${key}`); + const descriptors = Object.getOwnPropertyDescriptors(candidate); + // A null prototype makes __proto__ ordinary JSON data. Defining each + // key also avoids every inherited setter, so it cannot disappear or + // change the detached record's prototype before exact-key validation. + const output = Object.create(null) as Record; + for (const [key, descriptor] of Object.entries(descriptors)) { + if (!('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}.${key} must be an own enumerable data property`); + } + Object.defineProperty(output, key, { + value: copy(descriptor.value, `${path}.${key}`), + enumerable: true, + configurable: true, + writable: true, + }); + } + return output; + } finally { + activeAncestors.delete(candidate); } - return output; }; return deepFreezeFixedTrace(copy(value, label)); } diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index f8f5c162f2..dde74cd0f6 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { createHash } from 'node:crypto'; import { fixedTraceEstimatedCostUsd } from '../../../src/addie/eval/fixed-trace-budget.js'; import { FIXED_TRACE_CONFIRMATORY_POWER_GATE, FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, evaluateFixedTraceConfirmatoryClaim, fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; +import { snapshotFixedTraceJson } from '../../../src/addie/eval/fixed-trace-safe-snapshot.js'; function historicalOwnEnumerableFingerprint(value: unknown): string { const canonical = (current: unknown): string => { @@ -128,6 +129,53 @@ describe('fixed-trace evaluation protocol projection', () => { } }); + it('keeps prototype-shaped JSON as visible data and rejects it at every protocol fingerprint boundary', () => { + const hostile = JSON.parse('{"__proto__":{"polluted":true}}'); + const detached = snapshotFixedTraceJson(hostile, 'hostile JSON') as Record; + expect(Object.getPrototypeOf(detached)).toBe(null); + expect(Object.keys(detached)).toEqual(['__proto__']); + expect(Object.getOwnPropertyDescriptor(detached, '__proto__')?.value).toEqual({ polluted: true }); + expect(JSON.stringify(detached)).toBe('{"__proto__":{"polluted":true}}'); + expect(({} as { polluted?: boolean }).polluted).toBeUndefined(); + + for (const key of ['__proto__', 'prototype', 'constructor']) { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + Object.defineProperty(protocol, key, { enumerable: true, value: { poisoned: true } }); + expect(() => fixedTraceEvaluationProtocolFingerprint(protocol)).toThrow('dangerous prototype key'); + } + }); + + it('rejects inherited keys, symbols, accessors, Proxies, array extras, and cycles without mutating the snapshot', () => { + const inherited = Object.create({ inherited: true }); + expect(() => snapshotFixedTraceJson(inherited, 'inherited')).toThrow('plain object'); + + const symbol = { safe: true }; + Object.defineProperty(symbol, Symbol('hidden'), { enumerable: true, value: true }); + expect(() => snapshotFixedTraceJson(symbol, 'symbol')).toThrow('without symbols'); + + let reads = 0; + const accessor = {}; + Object.defineProperty(accessor, 'value', { enumerable: true, get() { reads += 1; return true; } }); + expect(() => snapshotFixedTraceJson(accessor, 'accessor')).toThrow('own enumerable data'); + expect(reads).toBe(0); + expect(() => snapshotFixedTraceJson(new Proxy({}, {}), 'proxy')).toThrow('Proxy'); + + const arrayExtra: any[] & { extra?: boolean } = [true]; + arrayExtra.extra = true; + expect(() => snapshotFixedTraceJson(arrayExtra, 'array extra')).toThrow('extra array property'); + + const cycle: { self?: unknown } = {}; + cycle.self = cycle; + expect(() => snapshotFixedTraceJson(cycle, 'cycle')).toThrow('cycle'); + + const mutable = { nested: { value: 1 } }; + const detached = snapshotFixedTraceJson(mutable, 'mutable') as { nested: { value: number } }; + mutable.nested.value = 2; + expect(detached.nested.value).toBe(1); + expect(Object.isFrozen(detached)).toBe(true); + expect(Object.isFrozen(detached.nested)).toBe(true); + }); + it('uses a detached closed snapshot for validation, hashing, and estimates', () => { const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); const expectedFingerprint = fixedTraceEvaluationProtocolFingerprint(protocol); diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts index c1e0aa5a38..2ebc3ea42d 100644 --- a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts +++ b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, estimateFixedTraceExperiment, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; +import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, estimateFixedTraceExperiment, fixedTraceCandidatePlanFingerprint, fixedTraceExperimentPlanFingerprint, fixedTraceTrustedManifestFingerprint, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; import { FIXED_TRACE_PARTITION_MANIFEST, FIXED_TRACE_PARTITION_MANIFEST_SHA256, FIXED_TRACE_PARTITION_MANIFEST_VERSION } from '../../../src/addie/eval/fixed-trace-partition.js'; import { CLAUDE_PRICING_VERSION } from '../../../src/addie/claude-pricing.js'; import { CODE_VERSION } from '../../../src/addie/config-version.js'; @@ -26,6 +26,19 @@ describe('fixed-trace experiment plan offline boundary', () => { const terra = plan(); terra.arms[0].router!.model = 'gpt-5.6-terra'; expect(() => validateFixedTraceExperimentPlanOffline(terra)).toThrow('Unavailable immutable pricing'); }); + it('does not lose prototype-pollution keys at plan and raw-ledger boundaries', () => { + for (const key of ['__proto__', 'prototype', 'constructor']) { + const hostile = plan() as any; + Object.defineProperty(hostile, key, { enumerable: true, value: { poisoned: true } }); + expect(() => fixedTraceExperimentPlanFingerprint(hostile, () => null)).toThrow('dangerous prototype key'); + expect(() => fixedTraceCandidatePlanFingerprint(hostile)).toThrow('dangerous prototype key'); + } + const manifest = { id: 'clean' } as any; + const hostileManifest = { id: 'clean' } as any; + Object.defineProperty(hostileManifest, '__proto__', { enumerable: true, value: { poisoned: true } }); + expect(fixedTraceTrustedManifestFingerprint(hostileManifest)) + .not.toBe(fixedTraceTrustedManifestFingerprint(manifest)); + }); it('does not invoke a hostile getter before rejecting it, and detaches estimates', () => { const hostile = plan() as any; let reads = 0; @@ -45,6 +58,9 @@ describe('fixed-trace experiment plan offline boundary', () => { const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, phaseId: 'router_only_screen' as const, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, callIndex: 1 as const, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); const ledger = { version: 'addie-fixed-trace-raw-ledger-v1' as const, trustedManifestSha256: HASH, planFingerprint: validateFixedTraceExperimentPlanOffline(current).planFingerprint, budgetIdentitySha256: estimateFixedTraceExperiment(current, () => null).budgetIdentitySha256, entries }; expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).not.toThrow(); + const hostileLedger = { ...ledger } as any; + Object.defineProperty(hostileLedger, '__proto__', { enumerable: true, value: { poisoned: true } }); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, hostileLedger, HASH)).toThrow('dangerous prototype key'); ledger.entries[1].sequence = 1; expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('sequence'); ledger.entries[1].sequence = 2; ledger.entries[0].exactToolNames = ['tampered']; From c00320a77f73869bc632679ada7220d59d9160ac Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 18:34:29 +0000 Subject: [PATCH 11/16] fix(addie): lock evaluator stage configurations --- .../eval/fixed-trace-evaluation-protocol.ts | 155 +++++++++++------- .../fixed-trace-evaluation-protocol.test.ts | 84 +++++++++- 2 files changed, 176 insertions(+), 63 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index 5104beb651..bcaf09554e 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -146,6 +146,12 @@ export interface FixedTraceProtocolStage { cacheMode: 'disabled'; } +const STAGE_FIELD_KEYS = Object.freeze([ + 'role', 'provider', 'model', 'reasoningEffort', 'pricingProfileId', + 'maxInputTokensPerInvocation', 'maxOutputTokensPerInvocation', 'timeoutMs', + 'maxInvocationsPerCase', 'transportRetries', 'samplingMode', 'temperature', 'cacheMode', +] as const); + export interface FixedTraceProtocolArm { id: string; architecture: FixedTraceProtocolArchitecture; @@ -372,11 +378,7 @@ function pricing(profileId: string, pricingAsOf: string): FixedTraceProtocolPric } function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: string): FixedTraceProtocolPricingProfile { - assertExactKeys(stage, [ - 'role', 'provider', 'model', 'reasoningEffort', 'pricingProfileId', - 'maxInputTokensPerInvocation', 'maxOutputTokensPerInvocation', 'timeoutMs', - 'maxInvocationsPerCase', 'transportRetries', 'samplingMode', 'temperature', 'cacheMode', - ], label); + assertExactKeys(stage, STAGE_FIELD_KEYS, label); positiveInteger(stage.maxInputTokensPerInvocation, `${label}.maxInputTokensPerInvocation`); positiveInteger(stage.maxOutputTokensPerInvocation, `${label}.maxOutputTokensPerInvocation`); positiveInteger(stage.timeoutMs, `${label}.timeoutMs`); @@ -397,52 +399,115 @@ function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: * Execution limits are evaluator-owned planning inputs, not caller-selected * estimates. Keep this matrix independent of the proposed protocol object so * a detached protocol supplied to an offline estimator cannot rewrite its - * phase, admission, result-use, or stop conditions. + * phase, arm, admission, result-use, or execution configuration. The private + * stage records are the only allowed price-bearing configurations; callers + * may provide JSON that equals them, but cannot select a price cohort. */ +const PRICE = Object.freeze({ + haiku: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, + sonnet: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, + gemini: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, +}); + +const router = ( + provider: ModelProviderId, + model: string, + reasoningEffort: ModelReasoningEffort, + pricingProfileId: string, +): FixedTraceProtocolStage => Object.freeze({ + role: 'router', provider, model, reasoningEffort, pricingProfileId, + maxInputTokensPerInvocation: 4_096, maxOutputTokensPerInvocation: 300, + timeoutMs: 120_000, maxInvocationsPerCase: 1, transportRetries: 0, + samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', +}); + +const generation = ( + provider: ModelProviderId, + model: string, + reasoningEffort: ModelReasoningEffort, + pricingProfileId: string, +): FixedTraceProtocolStage => Object.freeze({ + role: 'generation', provider, model, reasoningEffort, pricingProfileId, + maxInputTokensPerInvocation: 16_384, maxOutputTokensPerInvocation: 900, + timeoutMs: 120_000, maxInvocationsPerCase: 12, transportRetries: 0, + samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', +}); + const EVALUATOR_OWNED_PHASE_MATRIX = Object.freeze([ Object.freeze({ id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, arms: Object.freeze([Object.freeze({ - id: 'smoke-incumbent-two-stage', admission: 'planning_only', - stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', + stages: Object.freeze([ + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + ]), })]), }), Object.freeze({ id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, arms: Object.freeze([Object.freeze({ - id: 'router-haiku-default', admission: 'planning_only', - stopConditions: Object.freeze([['router', 1]] as const), + id: 'router-haiku-default', architecture: 'two_stage_llm_router', admission: 'planning_only', + stages: Object.freeze([router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]), })]), }), Object.freeze({ id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, arms: Object.freeze([Object.freeze({ - id: 'oracle-sonnet-default', admission: 'planning_only', - stopConditions: Object.freeze([['generation', 12]] as const), + id: 'oracle-sonnet-default', architecture: 'oracle_route_diagnostic', admission: 'planning_only', + stages: Object.freeze([generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet)]), })]), }), Object.freeze({ id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, arms: Object.freeze([ Object.freeze({ - id: 'incumbent-haiku-sonnet', admission: 'planning_only', - stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + id: 'incumbent-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', + stages: Object.freeze([ + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + ]), }), Object.freeze({ - id: 'gemini-low-medium-pipeline', admission: 'planning_only', - stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + id: 'gemini-low-medium-pipeline', architecture: 'two_stage_llm_router', admission: 'planning_only', + stages: Object.freeze([ + router('google', 'gemini-3.7-flash', 'low', PRICE.gemini), + generation('google', 'gemini-3.7-flash', 'medium', PRICE.gemini), + ]), }), ]), }), Object.freeze({ id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, arms: Object.freeze([Object.freeze({ - id: 'tuning-incumbent-haiku-sonnet', admission: 'planning_only', - stopConditions: Object.freeze([['router', 1], ['generation', 12]] as const), + id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', + stages: Object.freeze([ + router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + ]), })]), }), ] as const); +function matchesEvaluatorOwnedStage( + stage: FixedTraceProtocolStage, + expected: FixedTraceProtocolStage, +): boolean { + return stage.role === expected.role + && stage.provider === expected.provider + && stage.model === expected.model + && stage.reasoningEffort === expected.reasoningEffort + && stage.pricingProfileId === expected.pricingProfileId + && stage.maxInputTokensPerInvocation === expected.maxInputTokensPerInvocation + && stage.maxOutputTokensPerInvocation === expected.maxOutputTokensPerInvocation + && stage.timeoutMs === expected.timeoutMs + && stage.maxInvocationsPerCase === expected.maxInvocationsPerCase + && stage.transportRetries === expected.transportRetries + && stage.samplingMode === expected.samplingMode + && stage.temperature === expected.temperature + && stage.cacheMode === expected.cacheMode; +} + function assertEvaluatorOwnedPhaseMatrix(phase: FixedTraceProtocolPhase, index: number): void { const expected = EVALUATOR_OWNED_PHASE_MATRIX[index]; if (!expected @@ -458,16 +523,22 @@ function assertEvaluatorOwnedPhaseMatrix(phase: FixedTraceProtocolPhase, index: for (let armIndex = 0; armIndex < phase.arms.length; armIndex += 1) { const arm = phase.arms[armIndex]; const expectedArm = expected.arms[armIndex]; - if (!expectedArm || arm.id !== expectedArm.id || arm.admission !== expectedArm.admission) { - throw new Error(`${phase.id} arm does not match the evaluator-owned admission matrix`); + if (!expectedArm + || arm.id !== expectedArm.id + || arm.architecture !== expectedArm.architecture + || arm.admission !== expectedArm.admission) { + throw new Error(`${phase.id} arm does not match the evaluator-owned arm matrix`); } - if (arm.stages.length !== expectedArm.stopConditions.length || arm.stages.some((stage, stageIndex) => { - const expectedStop = expectedArm.stopConditions[stageIndex]; - return !expectedStop - || stage.role !== expectedStop[0] - || stage.maxInvocationsPerCase !== expectedStop[1]; - })) { - throw new Error(`${phase.id}.${arm.id} does not match the evaluator-owned stop-condition matrix`); + if (arm.stages.length !== expectedArm.stages.length) { + throw new Error(`${phase.id}.${arm.id} does not match the evaluator-owned stage configuration matrix`); + } + for (let stageIndex = 0; stageIndex < arm.stages.length; stageIndex += 1) { + const stage = arm.stages[stageIndex]; + const expectedStage = expectedArm.stages[stageIndex]; + assertExactKeys(stage, STAGE_FIELD_KEYS, `${phase.id}.${arm.id}.stage[${stageIndex}]`); + if (!expectedStage || !matchesEvaluatorOwnedStage(stage, expectedStage)) { + throw new Error(`${phase.id}.${arm.id} does not match the evaluator-owned stage configuration matrix`); + } } } } @@ -680,36 +751,6 @@ export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluat }); } -const router = ( - provider: ModelProviderId, - model: string, - reasoningEffort: ModelReasoningEffort, - pricingProfileId: string, -): FixedTraceProtocolStage => ({ - role: 'router', provider, model, reasoningEffort, pricingProfileId, - maxInputTokensPerInvocation: 4_096, maxOutputTokensPerInvocation: 300, - timeoutMs: 120_000, maxInvocationsPerCase: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', -}); - -const generation = ( - provider: ModelProviderId, - model: string, - reasoningEffort: ModelReasoningEffort, - pricingProfileId: string, -): FixedTraceProtocolStage => ({ - role: 'generation', provider, model, reasoningEffort, pricingProfileId, - maxInputTokensPerInvocation: 16_384, maxOutputTokensPerInvocation: 900, - timeoutMs: 120_000, maxInvocationsPerCase: 12, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', -}); - -const PRICE = Object.freeze({ - haiku: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, - sonnet: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, - gemini: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, -}); - /** Unsupported model names are inert metadata, never a stage or a price. */ export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ Object.freeze({ provider: 'openai' as const, model: 'gpt-5.6-terra', dispatchable: false as const, trustedPrice: null }), diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index dde74cd0f6..865a302ae5 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -64,13 +64,13 @@ describe('fixed-trace evaluation protocol projection', () => { expect(FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES).toEqual([{ provider: 'openai', model: 'gpt-5.6-terra', dispatchable: false, trustedPrice: null }, { provider: 'openai', model: 'gpt-5.6-sol', dispatchable: false, trustedPrice: null }]); const terra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); terra.phases[1].arms[0].stages[0].provider = 'openai'; terra.phases[1].arms[0].stages[0].model = 'gpt-5.6-terra'; - expect(() => assertFixedTraceEvaluationProtocol(terra)).toThrow('pricing profile does not match'); + expect(() => assertFixedTraceEvaluationProtocol(terra)).toThrow('evaluator-owned stage configuration matrix'); }); it('rejects reversed, duplicated, direct, smoke-promotion, and fabricated trust', () => { const reversed = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); reversed.phases.reverse(); expect(() => assertFixedTraceEvaluationProtocol(reversed)).toThrow('exact required order'); const direct = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); direct.phases[3].arms[0].architecture = 'direct_bounded_production_shaped'; - expect(() => assertFixedTraceEvaluationProtocol(direct)).toThrow('direct and hybrid'); + expect(() => assertFixedTraceEvaluationProtocol(direct)).toThrow('evaluator-owned arm matrix'); const promotional = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; promotional.phases[0].resultUse = 'promotional'; expect(() => assertFixedTraceEvaluationProtocol(promotional)).toThrow(); expect(() => assertFixedTraceEvaluationProtocolTrusted(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any)).toThrow('locked'); @@ -102,7 +102,7 @@ describe('fixed-trace evaluation protocol projection', () => { const substituted = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; substituted.phases[3].arms[1] = structuredClone(substituted.phases[3].arms[0]); - expect(() => estimateFixedTraceEvaluationProtocol(substituted)).toThrow('evaluator-owned admission matrix'); + expect(() => estimateFixedTraceEvaluationProtocol(substituted)).toThrow('evaluator-owned arm matrix'); }); it('enforces evaluator-owned admission, result use, counts, repetitions, and stop conditions for every phase', () => { @@ -119,16 +119,87 @@ describe('fixed-trace evaluation protocol projection', () => { for (let armIndex = 0; armIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms.length; armIndex += 1) { const admission = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; admission.phases[phaseIndex].arms[armIndex].admission = 'caller_promotional'; - expect(() => estimateFixedTraceEvaluationProtocol(admission)).toThrow('evaluator-owned admission matrix'); + expect(() => estimateFixedTraceEvaluationProtocol(admission)).toThrow('evaluator-owned arm matrix'); for (let stageIndex = 0; stageIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms[armIndex].stages.length; stageIndex += 1) { const stopCondition = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; stopCondition.phases[phaseIndex].arms[armIndex].stages[stageIndex].maxInvocationsPerCase = 999; - expect(() => estimateFixedTraceEvaluationProtocol(stopCondition)).toThrow('evaluator-owned stop-condition matrix'); + expect(() => estimateFixedTraceEvaluationProtocol(stopCondition)).toThrow('evaluator-owned stage configuration matrix'); } } } }); + it('prices only exact evaluator-owned provider, model, and execution configurations', () => { + const profile = (provider: 'anthropic' | 'google', model: string) => + FIXED_TRACE_PROTOCOL_PRICING.find((candidate) => candidate.provider === provider && candidate.model === model)!; + const haiku = profile('anthropic', 'claude-haiku-4-5'); + const sonnet = profile('anthropic', 'claude-sonnet-5'); + const gemini = profile('google', 'gemini-3.7-flash'); + const reject = (mutate: (stage: any) => void) => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + mutate(protocol.phases[1].arms[0].stages[0]); + expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow('evaluator-owned stage configuration matrix'); + }; + + reject((stage) => { stage.provider = gemini.provider; stage.model = gemini.model; stage.pricingProfileId = gemini.profileId; }); + reject((stage) => { stage.model = sonnet.model; stage.pricingProfileId = sonnet.profileId; }); + reject((stage) => { stage.model = 'claude-haiku-4.5'; }); + reject((stage) => { stage.pricingProfileId = sonnet.profileId; }); + expect(haiku.profileId).not.toBe(gemini.profileId); + + for (let phaseIndex = 0; phaseIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.length; phaseIndex += 1) { + for (let armIndex = 0; armIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms.length; armIndex += 1) { + for (let stageIndex = 0; stageIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms[armIndex].stages.length; stageIndex += 1) { + for (const mutate of [ + (stage: any) => { stage.reasoningEffort = stage.reasoningEffort === 'low' ? 'medium' : 'low'; }, + (stage: any) => { stage.maxInputTokensPerInvocation += 1; }, + (stage: any) => { stage.maxOutputTokensPerInvocation += 1; }, + (stage: any) => { stage.timeoutMs += 1; }, + (stage: any) => { stage.maxInvocationsPerCase += 1; }, + (stage: any) => { stage.transportRetries = 1; }, + (stage: any) => { stage.cacheMode = 'caller_cache'; }, + (stage: any) => { stage.samplingMode = 'caller_sampling'; }, + (stage: any) => { stage.temperature = 0; }, + ]) { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + mutate(protocol.phases[phaseIndex].arms[armIndex].stages[stageIndex]); + expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow('evaluator-owned stage configuration matrix'); + } + } + } + } + }); + + it('rejects missing, extra, reordered, duplicated, and hostile stage records before pricing', () => { + const reject = (mutate: (protocol: any) => void, message = 'evaluator-owned stage configuration matrix') => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + mutate(protocol); + expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow(message); + }; + reject((protocol) => { protocol.phases[0].arms[0].stages.pop(); }); + reject((protocol) => { protocol.phases[0].arms[0].stages.push(structuredClone(protocol.phases[0].arms[0].stages[0])); }); + reject((protocol) => { protocol.phases[0].arms[0].stages.reverse(); }); + reject((protocol) => { protocol.phases[0].arms[0].stages[1] = structuredClone(protocol.phases[0].arms[0].stages[0]); }); + + reject((protocol) => { + const stage = protocol.phases[1].arms[0].stages[0]; + const { provider: ignoredProvider, ...own } = stage; + void ignoredProvider; + protocol.phases[1].arms[0].stages[0] = Object.assign(Object.create({ provider: 'google' }), own); + }, 'plain object'); + reject((protocol) => { + Object.defineProperty(protocol.phases[1].arms[0].stages[0], 'provider', { + enumerable: true, + get() { return 'google'; }, + }); + }, 'own enumerable data'); + reject((protocol) => { protocol.phases[1].arms[0].stages[0] = new Proxy(protocol.phases[1].arms[0].stages[0], {}); }, 'Proxy'); + reject((protocol) => { Object.setPrototypeOf(protocol.phases[1].arms[0].stages[0], { provider: 'google' }); }, 'plain object'); + reject((protocol) => { + Object.defineProperty(protocol.phases[1].arms[0].stages[0], '__proto__', { enumerable: true, value: { poisoned: true } }); + }, 'dangerous prototype key'); + }); + it('keeps prototype-shaped JSON as visible data and rejects it at every protocol fingerprint boundary', () => { const hostile = JSON.parse('{"__proto__":{"polluted":true}}'); const detached = snapshotFixedTraceJson(hostile, 'hostile JSON') as Record; @@ -184,7 +255,8 @@ describe('fixed-trace evaluation protocol projection', () => { expect(estimate.stages.find((stage) => stage.phaseId === 'router_screen')?.outputTokenCeiling).toBe(46 * 3 * 300); expect(Object.isFrozen(estimate)).toBe(true); expect(Object.isFrozen(estimate.phases)).toBe(true); - expect(expectedFingerprint).not.toBe(fixedTraceEvaluationProtocolFingerprint(protocol)); + expect(expectedFingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(() => fixedTraceEvaluationProtocolFingerprint(protocol)).toThrow('evaluator-owned stage configuration matrix'); const arrayExtra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; arrayExtra.phases.extra = true; From e05d7f94423505eb84898cf2e83b3faaf376bedb Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 18:38:20 +0000 Subject: [PATCH 12/16] fix(addie): preserve holdout finalization gate --- .../addie/eval/fixed-trace-experiment-plan.ts | 17 +++++++++----- .../addie/fixed-trace-experiment-plan.test.ts | 22 ++++++++++++++++++- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts index 91908ea507..3fde0fcf1d 100644 --- a/server/src/addie/eval/fixed-trace-experiment-plan.ts +++ b/server/src/addie/eval/fixed-trace-experiment-plan.ts @@ -460,7 +460,6 @@ function assertPlanShape(plan: FixedTraceExperimentPlan): void { function assertFixedTraceExperimentPlanStructure( plan: FixedTraceExperimentPlan, - holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, ): void { assertPlanShape(plan); assertFixedTracePartitionManifest(); @@ -479,7 +478,7 @@ function assertFixedTraceExperimentPlanStructure( throw new Error('Experiment plan uses an uncommitted fixed-trace partition manifest'); } if (plan.partition.selected === 'holdout') { - assertHoldoutFinalization(plan, holdoutFinalizationResolver); + assertHoldoutFinalizationGateShape(plan); } else if (plan.partition.selected !== 'development' || plan.partition.finalizationGate) { throw new Error('Development execution must not carry a holdout finalization gate'); } @@ -650,10 +649,8 @@ function assertHoldoutFinalization( resolver: FixedTraceHoldoutFinalizationResolver | undefined, ): void { if (plan.partition.selected !== 'holdout') return; - const gate = plan.partition.finalizationGate; - if (!gate || gate.version !== FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION) { - throw new Error('Holdout is locked; an explicit versioned finalization gate is required'); - } + assertHoldoutFinalizationGateShape(plan); + const gate = plan.partition.finalizationGate!; if (!resolver) throw new Error('Holdout is locked; an externally resolved finalization record is required'); const record = resolver(gate.recordId); if (!record) throw new Error(`Holdout finalization record is unavailable: ${gate.recordId}`); @@ -665,6 +662,14 @@ function assertHoldoutFinalization( if (record.consumed) throw new Error('Holdout finalization record has already been consumed'); } +/** Shape validation is resolver-free so candidate fingerprints cannot recurse. */ +function assertHoldoutFinalizationGateShape(plan: FixedTraceExperimentPlan): void { + const gate = plan.partition.finalizationGate; + if (!gate || gate.version !== FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION) { + throw new Error('Holdout is locked; an explicit versioned finalization gate is required'); + } +} + export function assertFixedTraceExperimentPlan( plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts index 2ebc3ea42d..e11e7e9725 100644 --- a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts +++ b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, estimateFixedTraceExperiment, fixedTraceCandidatePlanFingerprint, fixedTraceExperimentPlanFingerprint, fixedTraceTrustedManifestFingerprint, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; +import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, assertFixedTraceExperimentPlan, estimateFixedTraceExperiment, fixedTraceCandidatePlanFingerprint, fixedTraceExperimentPlanFingerprint, fixedTraceTrustedManifestFingerprint, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; import { FIXED_TRACE_PARTITION_MANIFEST, FIXED_TRACE_PARTITION_MANIFEST_SHA256, FIXED_TRACE_PARTITION_MANIFEST_VERSION } from '../../../src/addie/eval/fixed-trace-partition.js'; import { CLAUDE_PRICING_VERSION } from '../../../src/addie/claude-pricing.js'; import { CODE_VERSION } from '../../../src/addie/config-version.js'; @@ -53,6 +53,26 @@ describe('fixed-trace experiment plan offline boundary', () => { (mutable.arms as any).extra = true; expect(() => validateFixedTraceExperimentPlanOffline(mutable)).toThrow('extra array property'); }); + it('validates a declared holdout gate before its resolver check without recursive fingerprinting', () => { + const holdout = plan() as any; + holdout.partition = { + manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, + manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, + selected: 'holdout', + finalizationGate: { version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, recordId: 'finalize-1' }, + }; + holdout.arms[0].router.requestBounds.inputBytesByTrace = Object.fromEntries( + FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100]]), + ); + const fingerprint = fixedTraceCandidatePlanFingerprint(holdout); + expect(fingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(() => assertFixedTraceExperimentPlan(holdout, () => null, () => ({ + version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, + trustedManifestId: holdout.trustedManifestId, + frozenCandidatePlanFingerprint: fingerprint, + consumed: false, + }))).toThrow('Trusted fixed-trace manifest is locked'); + }); it('requires exact ledger sequence, tools, and offline provider resolution', () => { const current = plan(); const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, phaseId: 'router_only_screen' as const, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, callIndex: 1 as const, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); From e8ab11e2493dc088fbc9a364976a4e1a0f6adc0c Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 20:17:32 +0000 Subject: [PATCH 13/16] fix(addie): repair fixed trace evaluation plan --- .../eval/fixed-trace-evaluation-protocol.ts | 210 +++++++++--- .../addie/eval/fixed-trace-experiment-plan.ts | 302 +++++------------- .../src/addie/eval/fixed-trace-partition.ts | 13 +- server/src/addie/model-cost-pricing.ts | 22 ++ .../addie/fixed-trace-diagnostic-cli.test.ts | 4 +- .../fixed-trace-evaluation-protocol.test.ts | 53 ++- .../addie/fixed-trace-experiment-plan.test.ts | 49 +-- 7 files changed, 358 insertions(+), 295 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index bcaf09554e..16fcf2eb52 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -2,8 +2,10 @@ import { createHash } from 'node:crypto'; import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_LUNA_PRICING_VERSION, } from '../model-cost-pricing.js'; import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; +import { OPENAI_ROUTER_MODEL } from '../model-providers/openai-responses-provider.js'; import { fixedTraceEstimatedCostUsd, validateFixedTracePricing, @@ -16,7 +18,7 @@ import { deepFreezeFixedTrace, snapshotFixedTraceJson } from './fixed-trace-safe /** * A planning-only contract. It has no dispatcher and is deliberately unable - * to make a corpus, an execution envelope, or a sealed holdout trusted. + * to make a corpus, an execution envelope, or a confirmatory final pack trusted. */ export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = 'addie-fixed-trace-evaluation-protocol-v1' as const; @@ -24,26 +26,32 @@ export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = /** * Evaluator-owned confirmatory precision rule. The conservative normal * approximation assumes the maximum possible variance (1) of a paired - * case-level difference in [-1, 1], with one-sided alpha .025 and 80% power. - * The non-inferiority margin is limiting: ceil((1.9599639845 + .8416212336)^2 - * / .03^2) = 8,721 independent paired cases. This is deliberately a sample - * requirement, not a price quote or an authorization to spend. + * case-level difference in [-1, 1]. The primary family is exactly two + * one-sided claims (superiority and non-inferiority). Holm's first rejection + * is allocated alpha .0125, which yields the conservative normal-approximate + * requirements below. An evaluator-owned exact paired-discordance power + * calculation remains required before a confirmatory claim. */ export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ version: 'addie-fixed-trace-confirmatory-power-v1', unit: 'unique_paired_case', repetitionsCountAsIndependentCases: false, - oneSidedAlpha: 0.025, + primaryHypothesisFamily: Object.freeze({ + size: 2, + correction: 'holm', + orderedOneSidedAlpha: Object.freeze([0.0125, 0.025]), + }), targetPower: 0.8, conservativePairedDifferenceVarianceUpperBound: 1, superiorityMarginPercentagePoints: 5, nonInferiorityMarginPercentagePoints: -3, - superiorityRequiredIndependentEvaluableCases: 3_140, - nonInferiorityRequiredIndependentEvaluableCases: 8_721, - requiredIndependentEvaluableCases: 8_721, + superiorityRequiredIndependentEvaluableCases: 3_803, + nonInferiorityRequiredIndependentEvaluableCases: 10_562, + requiredIndependentEvaluableCases: 10_562, requiredAnalysis: Object.freeze({ resampling: 'grouped_stratified_case_level_bootstrap', multiplicityCorrection: 'holm', + pairedDiscordancePower: 'evaluator_owned_exact_paired_discordance_contract_unavailable', pairedDiscordanceTest: 'predeclared_exact_paired_test_required', }), currentScreeningTuningUniqueCaseCount: 120, @@ -84,6 +92,20 @@ export interface FixedTraceProtocolPricingProfile extends FixedTracePricing { * must add a reviewed ceiling instead of silently reusing these values. */ export const FIXED_TRACE_PROTOCOL_PRICING = Object.freeze([ + Object.freeze({ + provider: 'openai', + model: OPENAI_ROUTER_MODEL, + profileId: `${OPENAI_GPT_5_6_LUNA_PRICING_VERSION}:${OPENAI_ROUTER_MODEL}`, + version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', + inputUsdPerMillionTokens: 0.2, + outputUsdPerMillionTokens: 1.2, + cacheReadUsdPerMillionTokens: null, + cacheWriteUsdPerMillionTokens: null, + cacheReadAccounting: 'unsupported', + cacheWriteAccounting: 'unsupported', + source: 'Repository OpenAI Luna router price pin, checked 2026-08-26.', + }), Object.freeze({ provider: 'anthropic', model: 'claude-haiku-4-5', @@ -130,6 +152,8 @@ export const FIXED_TRACE_PROTOCOL_PRICING = Object.freeze([ export interface FixedTraceProtocolStage { role: FixedTraceProtocolStageRole; + /** Judges receive blinded candidate artifacts; candidate stages do not. */ + blinded: true | null; provider: ModelProviderId; model: string; reasoningEffort: ModelReasoningEffort; @@ -147,7 +171,7 @@ export interface FixedTraceProtocolStage { } const STAGE_FIELD_KEYS = Object.freeze([ - 'role', 'provider', 'model', 'reasoningEffort', 'pricingProfileId', + 'role', 'blinded', 'provider', 'model', 'reasoningEffort', 'pricingProfileId', 'maxInputTokensPerInvocation', 'maxOutputTokensPerInvocation', 'timeoutMs', 'maxInvocationsPerCase', 'transportRetries', 'samplingMode', 'temperature', 'cacheMode', ] as const); @@ -156,10 +180,31 @@ export interface FixedTraceProtocolArm { id: string; architecture: FixedTraceProtocolArchitecture; admission: FixedTraceProtocolAdmission; + /** The three architecture arms share this frozen comparison universe. */ + ablationControlId: string | null; + /** Luna may judge only after an independently verified calibration admission. */ + lunaJudgeCalibration: 'not_applicable' | 'requires_verified_luna_judge_calibration'; /** Each judge appears once; exactly two are required for compared outputs. */ stages: readonly FixedTraceProtocolStage[]; } +/** + * Exactly what the architecture ablation holds fixed. These are contracts for + * a future evaluator, not authority to run one. The generator's stage record + * supplies the exact provider/model/effort/limits; all three final arms use + * that same record. + */ +export const FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL = Object.freeze({ + id: 'fixed-trace-architecture-ablation-v1', + cases: 'same_evaluator_owned_cases_and_order', + generator: 'same_anthropic_claude_sonnet_5_provider_default_stage', + promptToolUniverse: 'same_production_shaped_prompt_system_docs_tools_schemas', + simulatorReceipts: 'same_fixed_trace_simulator_receipts_and_result_provenance', + executionLimits: 'same_input_output_timeout_invocation_retry_cache_sampling_controls', + judging: 'same_two_blinded_provider_excluding_judges', + failureDenominator: 'same_all_planned_case_stage_invocations_including_failures', +} as const); + export interface FixedTraceProtocolPhase { id: FixedTraceProtocolPhaseId; uniqueCaseCount: number; @@ -255,7 +300,7 @@ export interface FixedTraceProtocolEstimate { expectedSpendUsd: null; stages: readonly FixedTraceProtocolStageEstimate[]; phases: readonly FixedTraceProtocolPhaseEstimate[]; - screening: { candidateCeilingUsd: number; judgeCeilingUsd: number; totalCeilingUsd: number }; + screening: { candidateCeilingUsd: number; judgeCeilingUsd: number; contingencyUsd: number; totalCeilingUsd: number }; unavailableFinalTarget: FixedTraceEvaluationProtocol['unavailableFinalTarget']; /** The confirmatory sample remains unpriced and cannot authorize spend. */ budgetProjection: { @@ -386,6 +431,9 @@ function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: if (stage.transportRetries !== 0 || stage.samplingMode !== 'provider_no_sampling_control' || stage.temperature !== null || stage.cacheMode !== 'disabled') { throw new Error(`${label} has an unsupported execution control`); } + if ((stage.role === 'judge' && stage.blinded !== true) || (stage.role !== 'judge' && stage.blinded !== null)) { + throw new Error(`${label} has an invalid blinded-judge control`); + } const resolved = pricing(stage.pricingProfileId, pricingAsOf); if ( resolved.profileId !== stage.pricingProfileId @@ -404,6 +452,7 @@ function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: * may provide JSON that equals them, but cannot select a price cohort. */ const PRICE = Object.freeze({ + luna: `${OPENAI_GPT_5_6_LUNA_PRICING_VERSION}:${OPENAI_ROUTER_MODEL}`, haiku: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, sonnet: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, gemini: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, @@ -415,7 +464,7 @@ const router = ( reasoningEffort: ModelReasoningEffort, pricingProfileId: string, ): FixedTraceProtocolStage => Object.freeze({ - role: 'router', provider, model, reasoningEffort, pricingProfileId, + role: 'router', blinded: null, provider, model, reasoningEffort, pricingProfileId, maxInputTokensPerInvocation: 4_096, maxOutputTokensPerInvocation: 300, timeoutMs: 120_000, maxInvocationsPerCase: 1, transportRetries: 0, samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', @@ -427,17 +476,34 @@ const generation = ( reasoningEffort: ModelReasoningEffort, pricingProfileId: string, ): FixedTraceProtocolStage => Object.freeze({ - role: 'generation', provider, model, reasoningEffort, pricingProfileId, + role: 'generation', blinded: null, provider, model, reasoningEffort, pricingProfileId, maxInputTokensPerInvocation: 16_384, maxOutputTokensPerInvocation: 900, timeoutMs: 120_000, maxInvocationsPerCase: 12, transportRetries: 0, samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', }); +const judge = ( + provider: ModelProviderId, + model: string, + reasoningEffort: ModelReasoningEffort, + pricingProfileId: string, +): FixedTraceProtocolStage => Object.freeze({ + role: 'judge', blinded: true, provider, model, reasoningEffort, pricingProfileId, + maxInputTokensPerInvocation: 16_384, maxOutputTokensPerInvocation: 300, + timeoutMs: 120_000, maxInvocationsPerCase: 1, transportRetries: 0, + samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', +}); + +const NO_ABLATION = null; +const NO_LUNA_JUDGE_CALIBRATION = 'not_applicable' as const; +const LUNA_JUDGE_CALIBRATION = 'requires_verified_luna_judge_calibration' as const; +const ABLATION = FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id; + const EVALUATOR_OWNED_PHASE_MATRIX = Object.freeze([ Object.freeze({ id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, arms: Object.freeze([Object.freeze({ - id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', + id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), @@ -448,31 +514,50 @@ const EVALUATOR_OWNED_PHASE_MATRIX = Object.freeze([ id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, arms: Object.freeze([Object.freeze({ id: 'router-haiku-default', architecture: 'two_stage_llm_router', admission: 'planning_only', + ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]), + }), Object.freeze({ + id: 'router-luna-none', architecture: 'two_stage_llm_router', admission: 'planning_only', + ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, + stages: Object.freeze([router('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna)]), })]), }), Object.freeze({ id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, arms: Object.freeze([Object.freeze({ - id: 'oracle-sonnet-default', architecture: 'oracle_route_diagnostic', admission: 'planning_only', + id: 'generator-sonnet-default', architecture: 'oracle_route_diagnostic', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet)]), + }), Object.freeze({ + id: 'generator-haiku-default', architecture: 'oracle_route_diagnostic', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, + stages: Object.freeze([generation('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]), })]), }), Object.freeze({ id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, arms: Object.freeze([ Object.freeze({ - id: 'incumbent-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', + id: 'routed-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), + judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), ]), }), Object.freeze({ - id: 'gemini-low-medium-pipeline', architecture: 'two_stage_llm_router', admission: 'planning_only', + id: 'safe-hybrid-sonnet', architecture: 'hybrid_safe_signal_then_llm', admission: 'requires_verified_hybrid_contract', ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ - router('google', 'gemini-3.7-flash', 'low', PRICE.gemini), - generation('google', 'gemini-3.7-flash', 'medium', PRICE.gemini), + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), + judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), + ]), + }), + Object.freeze({ + id: 'bounded-direct-sonnet', architecture: 'direct_bounded_production_shaped', admission: 'requires_verified_direct_contract', ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, + stages: Object.freeze([ + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), + judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), ]), }), ]), @@ -480,7 +565,7 @@ const EVALUATOR_OWNED_PHASE_MATRIX = Object.freeze([ Object.freeze({ id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, arms: Object.freeze([Object.freeze({ - id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', + id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), @@ -494,6 +579,7 @@ function matchesEvaluatorOwnedStage( expected: FixedTraceProtocolStage, ): boolean { return stage.role === expected.role + && stage.blinded === expected.blinded && stage.provider === expected.provider && stage.model === expected.model && stage.reasoningEffort === expected.reasoningEffort @@ -526,7 +612,9 @@ function assertEvaluatorOwnedPhaseMatrix(phase: FixedTraceProtocolPhase, index: if (!expectedArm || arm.id !== expectedArm.id || arm.architecture !== expectedArm.architecture - || arm.admission !== expectedArm.admission) { + || arm.admission !== expectedArm.admission + || arm.ablationControlId !== expectedArm.ablationControlId + || arm.lunaJudgeCalibration !== expectedArm.lunaJudgeCalibration) { throw new Error(`${phase.id} arm does not match the evaluator-owned arm matrix`); } if (arm.stages.length !== expectedArm.stages.length) { @@ -544,7 +632,7 @@ function assertEvaluatorOwnedPhaseMatrix(phase: FixedTraceProtocolPhase, index: } function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, pricingAsOf: string): void { - assertExactKeys(arm, ['id', 'architecture', 'admission', 'stages'], `protocol arm`); + assertExactKeys(arm, ['id', 'architecture', 'admission', 'ablationControlId', 'lunaJudgeCalibration', 'stages'], `protocol arm`); if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid protocol arm ID: ${arm.id}`); const routers = arm.stages.filter((stage) => stage.role === 'router'); const generations = arm.stages.filter((stage) => stage.role === 'generation'); @@ -557,7 +645,36 @@ function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, p return; } if (generations.length !== 1 || routers.length > 1) throw new Error(`${arm.id} requires exactly one generation stage and at most one router`); - if (arm.architecture === 'two_stage_llm_router') { + if (phase.id === 'deployable_architecture') { + if (arm.ablationControlId !== ABLATION) throw new Error(`${arm.id} must use the fixed architecture ablation control`); + const candidateProviders = new Set([...routers, ...generations].map((stage) => stage.provider)); + if (candidateProviders.size !== 1) throw new Error(`${arm.id} must be a single-provider candidate pipeline for independent judging`); + if (arm.architecture === 'two_stage_llm_router' && (routers.length !== 1 || arm.admission !== 'planning_only')) { + throw new Error(`${arm.id} must use its locked routed architecture contract`); + } + if (arm.architecture === 'hybrid_safe_signal_then_llm' && (routers.length !== 0 || arm.admission !== 'requires_verified_hybrid_contract')) { + throw new Error(`${arm.id} requires its verified hybrid admission`); + } + if (arm.architecture === 'direct_bounded_production_shaped' && (routers.length !== 0 || arm.admission !== 'requires_verified_direct_contract')) { + throw new Error(`${arm.id} requires its verified direct admission`); + } + if (!['two_stage_llm_router', 'hybrid_safe_signal_then_llm', 'direct_bounded_production_shaped'].includes(arm.architecture)) { + throw new Error(`${arm.id} is not an architecture-ablation candidate`); + } + if (judges.length !== 2 || arm.stages.slice(-2).some((stage) => stage.role !== 'judge')) { + throw new Error(`${arm.id} requires exactly two trailing blinded judges`); + } + const judgeProviders = new Set(judges.map((stage) => stage.provider)); + if (judgeProviders.size !== 2 || [...judgeProviders].some((provider) => candidateProviders.has(provider))) { + throw new Error(`${arm.id} judges must be provider-excluding and independent`); + } + const usesLunaJudge = judges.some((stage) => stage.provider === 'openai' && stage.model === OPENAI_ROUTER_MODEL); + if (usesLunaJudge !== (arm.lunaJudgeCalibration === LUNA_JUDGE_CALIBRATION)) { + throw new Error(`${arm.id} Luna judge calibration admission is not locked`); + } + } else if (arm.ablationControlId !== NO_ABLATION || arm.lunaJudgeCalibration !== NO_LUNA_JUDGE_CALIBRATION) { + throw new Error(`${arm.id} has architecture-ablation controls outside the ablation phase`); + } else if (arm.architecture === 'two_stage_llm_router') { if (routers.length !== 1) throw new Error(`${arm.id} requires a router stage`); } else if (arm.architecture !== 'oracle_route_diagnostic' || routers.length !== 0) { throw new Error(`${arm.id} direct and hybrid substitutions are not admitted`); @@ -566,7 +683,7 @@ function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, p throw new Error(`${arm.id} oracle routing is diagnostic-only`); } for (const stage of arm.stages) assertStage(stage, `${arm.id}.${stage.role}`, pricingAsOf); - if (judges.length !== 0) throw new Error(`${arm.id} judges are blocked in the diagnostic-only protocol`); + if (phase.id !== 'deployable_architecture' && judges.length !== 0) throw new Error(`${arm.id} judges are blocked outside the architecture ablation`); } function validatedProtocolSnapshot(protocol: FixedTraceEvaluationProtocol): FixedTraceEvaluationProtocol { @@ -716,11 +833,12 @@ export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluat const candidateCeilingUsd = phases.reduce((total, phase) => total + phase.candidateCeilingUsd, 0); const judgeCeilingUsd = phases.reduce((total, phase) => total + phase.judgeCeilingUsd, 0); const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * snapshot.contingencyBasisPoints / 10_000; - const summarize = (source: readonly FixedTraceProtocolPhaseEstimate[]) => Object.freeze({ - candidateCeilingUsd: source.reduce((total, phase) => total + phase.candidateCeilingUsd, 0), - judgeCeilingUsd: source.reduce((total, phase) => total + phase.judgeCeilingUsd, 0), - totalCeilingUsd: source.reduce((total, phase) => total + phase.totalCeilingUsd, 0), - }); + const summarize = (source: readonly FixedTraceProtocolPhaseEstimate[]) => { + const candidateCeilingUsd = source.reduce((total, phase) => total + phase.candidateCeilingUsd, 0); + const judgeCeilingUsd = source.reduce((total, phase) => total + phase.judgeCeilingUsd, 0); + const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * snapshot.contingencyBasisPoints / 10_000; + return Object.freeze({ candidateCeilingUsd, judgeCeilingUsd, contingencyUsd, totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd + contingencyUsd }); + }; return Object.freeze({ protocolFingerprint: sha256(snapshot), dispatchable: false, @@ -763,7 +881,9 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto id: 'addie-6842-6846-staged-v1', trustedManifestId: 'externally-owned-addie-fixed-trace-v120', pricingAsOf: '2026-09-05T12:00:00.000Z', - contingencyBasisPoints: 0, + // Includes explicit failure/usage accounting contingency in every reported + // screening ceiling; it is still a non-authorizing offline projection. + contingencyBasisPoints: 2_000, unavailableFinalTarget: { availability: 'unavailable', uniqueCaseCount: 38, repetitions: 3, missingCaseCount: 38, }, @@ -771,7 +891,7 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto Object.freeze({ id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, resultUse: 'diagnostic_only', arms: Object.freeze([Object.freeze({ - id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', + id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), @@ -780,29 +900,43 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto }), Object.freeze({ id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, resultUse: 'diagnostic_only', - arms: Object.freeze([Object.freeze({ id: 'router-haiku-default', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]) })]), + arms: Object.freeze([ + Object.freeze({ id: 'router-haiku-default', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]) }), + Object.freeze({ id: 'router-luna-none', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([router('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna)]) }), + ]), }), Object.freeze({ id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, resultUse: 'diagnostic_only', - arms: Object.freeze([Object.freeze({ id: 'oracle-sonnet-default', architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, stages: Object.freeze([generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet)]) })]), + arms: Object.freeze([ + Object.freeze({ id: 'generator-sonnet-default', architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet)]) }), + Object.freeze({ id: 'generator-haiku-default', architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([generation('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]) }), + ]), }), Object.freeze({ id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, resultUse: 'diagnostic_only', arms: Object.freeze([ - Object.freeze({ id: 'incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + Object.freeze({ id: 'routed-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), + judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), ]) }), - Object.freeze({ id: 'gemini-low-medium-pipeline', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ - router('google', 'gemini-3.7-flash', 'low', PRICE.gemini), - generation('google', 'gemini-3.7-flash', 'medium', PRICE.gemini), + Object.freeze({ id: 'safe-hybrid-sonnet', architecture: 'hybrid_safe_signal_then_llm' as const, admission: 'requires_verified_hybrid_contract' as const, ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), + judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), + ]) }), + Object.freeze({ id: 'bounded-direct-sonnet', architecture: 'direct_bounded_production_shaped' as const, admission: 'requires_verified_direct_contract' as const, ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ + generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), + judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), ]) }), ]), }), Object.freeze({ id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, resultUse: 'diagnostic_only', arms: Object.freeze([ - Object.freeze({ id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, stages: Object.freeze([ + Object.freeze({ id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), ]) }), ]), diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts index 3fde0fcf1d..1368e5b134 100644 --- a/server/src/addie/eval/fixed-trace-experiment-plan.ts +++ b/server/src/addie/eval/fixed-trace-experiment-plan.ts @@ -2,8 +2,10 @@ import { createHash } from 'node:crypto'; import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_LUNA_PRICING_VERSION, } from '../model-cost-pricing.js'; import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; +import { OPENAI_ROUTER_MODEL } from '../model-providers/openai-responses-provider.js'; import { FIXED_TRACE_PARTITION_MANIFEST, FIXED_TRACE_PARTITION_MANIFEST_SHA256, @@ -14,7 +16,6 @@ import type { AddieTool } from '../types.js'; import { CODE_VERSION } from '../config-version.js'; import { FIXED_TRACE_STAGE_CONTROL_VERSION, - FIXED_TRACE_SUITE, type FixedTraceCase, } from './fixed-trace-suite.js'; import type { FixedTraceToolDefinitionProvenance } from './fixed-trace-architecture.js'; @@ -22,11 +23,9 @@ import { deepFreezeFixedTrace, snapshotFixedTraceJson } from './fixed-trace-safe /** A versioned, network-free admission contract for fixed-trace experiments. */ export const FIXED_TRACE_EXPERIMENT_PLAN_VERSION = 'addie-fixed-trace-experiment-plan-v1' as const; -export const FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION = - 'addie-fixed-trace-holdout-finalization-v1' as const; export const FIXED_TRACE_RAW_LEDGER_VERSION = 'addie-fixed-trace-raw-ledger-v1' as const; -export const FIXED_TRACE_HOLDOUT_LIMITATION = - 'execution_locked_repository_visible_not_secret_holdout' as const; +export const FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION = + 'repository_visible_development_validation_not_confirmatory_holdout' as const; export type FixedTraceExperimentArchitecture = | 'two_stage_llm_router' @@ -53,6 +52,11 @@ export interface FixedTraceImmutablePricingProfile { * unavailable, rather than inheriting a sibling model's price. */ export const FIXED_TRACE_IMMUTABLE_PRICING = Object.freeze([ + Object.freeze({ + provider: 'openai', model: OPENAI_ROUTER_MODEL, version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, + validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, + source: 'Repository OpenAI Luna router price pin, checked 2026-08-26.', + }), Object.freeze({ provider: 'anthropic', model: 'claude-haiku-4-5', version: CLAUDE_PRICING_VERSION, validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, @@ -127,9 +131,7 @@ export interface FixedTraceExperimentPlan { partition: { manifestVersion: typeof FIXED_TRACE_PARTITION_MANIFEST_VERSION; manifestSha256: typeof FIXED_TRACE_PARTITION_MANIFEST_SHA256; - selected: 'development' | 'holdout'; - /** Holdout is legal only for a separately versioned, explicit finalization. */ - finalizationGate?: { version: typeof FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION; recordId: string }; + selected: 'development' | 'repository_visible_development_validation'; }; ordering: { seed: string }; budgets: { candidateCeilingUsd: number; judgeCeilingUsd: number }; @@ -151,7 +153,7 @@ export interface FixedTraceTrustedManifest { * Resolver-owned, phase-selected execution inputs. They are never inferred * from a plan or copied onto an observation after execution. */ - suites: Readonly>; + suites: Readonly>; partitionManifestSha256: string; rawLedgerVersion: typeof FIXED_TRACE_RAW_LEDGER_VERSION; gitCommit: string; @@ -198,21 +200,16 @@ export interface FixedTraceExperimentRunnerBinding { } /** - * Finalization state belongs to a controlled store, not the candidate plan. - * `consume` is intentionally separate from inspection: dry runs never spend - * the one-time holdout authorization. + * A confirmatory finalization authority is intentionally absent. The only + * in-repository secondary split is development validation, not a secret pack; + * an externally authored/custodied final pack needs its own reviewed system. */ -export interface FixedTraceHoldoutFinalizationRecord { - id: string; - version: typeof FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION; - trustedManifestId: string; - frozenCandidatePlanFingerprint: string; - consumed: boolean; - tracePackVisibility: 'repository_visible' | 'externally_sealed'; -} -export type FixedTraceHoldoutFinalizationResolver = (id: string) => FixedTraceHoldoutFinalizationRecord | null; -export type FixedTraceHoldoutFinalizationConsumer = (id: string, frozenCandidatePlanFingerprint: string) => boolean; +/** + * An untrusted proposed observation shape. It cannot be validated or used as + * evidence in this PR. A future evaluator must bind every field below to a + * `FixedTraceTrustedExecutionExpectation` before the matching invocation. + */ export interface FixedTraceRawLedgerEntry { sequence: number; /** The plan-controlled stage grouping; it cannot be supplied out of order. */ @@ -221,14 +218,19 @@ export interface FixedTraceRawLedgerEntry { repetitionIndex: number; traceId: string; stage: 'router' | 'generation' | 'judge'; - /** One ledger entry represents one configured stage invocation envelope. */ - callIndex: 1; + /** Monotonic configured invocation within a stage; never collapsed to one row. */ + callIndex: number; + /** Every network attempt is represented, including failed attempts. */ + attemptIndex: number; dispatched: boolean; requestedProvider: ModelProviderId | null; requestedModel: string | null; returnedProvider: ModelProviderId | null; returnedModel: string | null; promptSha256: string; + systemSha256: string; + docsSha256: string; + toolSchemaSha256: string; providerRequestSha256: string | null; responseSha256: string | null; /** Content-addressed immutable raw artifacts; their bytes stay outside summaries. */ @@ -238,6 +240,8 @@ export interface FixedTraceRawLedgerEntry { caseControlSha256: string; executionEnvelopeSha256: string; directAdmissionSha256: string; + simulatorReceiptSha256: string; + simulatorResultProvenanceSha256: string; maxOutputTokens: number | null; timeoutMs: number | null; maxIterations: number | null; @@ -245,6 +249,8 @@ export interface FixedTraceRawLedgerEntry { reasoningEffort: ModelReasoningEffort; samplingMode: 'temperature_zero' | 'provider_no_sampling_control' | null; cacheMode: 'disabled' | null; + pricingProfileId: string | null; + failureDenominatorId: string; /** Offline validation admits only the explicit non-dispatch terminal state. */ status: 'not_dispatched'; finishReason: null; @@ -252,6 +258,18 @@ export interface FixedTraceRawLedgerEntry { estimatedCostUsd: null; } +/** + * Evaluator-owned pre-dispatch expected values. A coordinator must create and + * authenticate this complete sequence from the immutable manifest before any + * provider call; there is deliberately no implementation in this PR. + */ +export interface FixedTraceTrustedExecutionExpectation { + trustedManifestSha256: string; + planFingerprint: string; + budgetIdentitySha256: string; + entries: readonly FixedTraceRawLedgerEntry[]; +} + export interface FixedTraceRawAuditableLedger { version: typeof FIXED_TRACE_RAW_LEDGER_VERSION; trustedManifestSha256: string; @@ -334,7 +352,9 @@ function requirePositiveInteger(value: number, label: string): void { } function selectedTraceIds(plan: FixedTraceExperimentPlan): readonly string[] { - return FIXED_TRACE_PARTITION_MANIFEST[plan.partition.selected]; + return plan.partition.selected === 'development' + ? FIXED_TRACE_PARTITION_MANIFEST.development + : FIXED_TRACE_PARTITION_MANIFEST.repositoryVisibleDevelopmentValidation; } function pricingFor(stage: FixedTracePlannedStage, pricingAsOf: string): FixedTraceImmutablePricingProfile { @@ -448,12 +468,7 @@ function assertPlanShape(plan: FixedTraceExperimentPlan): void { 'traceSuiteSha256', 'promptConfigVersion', 'toolSchemaSha256', 'toolDefinitionProvenance', 'providerDegradationInjectionEnabled', 'partition', 'ordering', 'budgets', 'arms', ], 'experiment plan'); - assertExactKeys(plan.partition, [ - 'manifestVersion', 'manifestSha256', 'selected', ...(plan.partition.finalizationGate ? ['finalizationGate'] : []), - ], 'experiment plan.partition'); - if (plan.partition.finalizationGate) { - assertExactKeys(plan.partition.finalizationGate, ['version', 'recordId'], 'experiment plan.partition.finalizationGate'); - } + assertExactKeys(plan.partition, ['manifestVersion', 'manifestSha256', 'selected'], 'experiment plan.partition'); assertExactKeys(plan.ordering, ['seed'], 'experiment plan.ordering'); assertExactKeys(plan.budgets, ['candidateCeilingUsd', 'judgeCeilingUsd'], 'experiment plan.budgets'); } @@ -477,10 +492,8 @@ function assertFixedTraceExperimentPlanStructure( if (plan.partition.manifestVersion !== FIXED_TRACE_PARTITION_MANIFEST_VERSION || plan.partition.manifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { throw new Error('Experiment plan uses an uncommitted fixed-trace partition manifest'); } - if (plan.partition.selected === 'holdout') { - assertHoldoutFinalizationGateShape(plan); - } else if (plan.partition.selected !== 'development' || plan.partition.finalizationGate) { - throw new Error('Development execution must not carry a holdout finalization gate'); + if (!['development', 'repository_visible_development_validation'].includes(plan.partition.selected)) { + throw new Error('Only repository-visible development partitions are available'); } if (!plan.ordering.seed.trim()) throw new Error('Experiment ordering seed is required'); if (!Number.isFinite(plan.budgets.candidateCeilingUsd) || plan.budgets.candidateCeilingUsd <= 0) throw new Error('candidateCeilingUsd must be positive'); @@ -527,67 +540,15 @@ export function validateFixedTraceRawAuditableLedgerOffline( ledger: FixedTraceRawAuditableLedger, expectedTrustedManifestSha256: string, ): void { - const safePlan = validatedPlanSnapshot(plan); - const safeLedger = snapshotFixedTraceJson(ledger, 'raw ledger') as FixedTraceRawAuditableLedger; - requireHash(expectedTrustedManifestSha256, 'expected trustedManifestSha256'); - assertExactKeys(safeLedger, ['version', 'trustedManifestSha256', 'planFingerprint', 'budgetIdentitySha256', 'entries'], 'raw ledger'); - if (safeLedger.version !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Unsupported raw fixed-trace ledger version'); - if (safeLedger.trustedManifestSha256 !== expectedTrustedManifestSha256) throw new Error('Raw ledger trusted manifest mismatch'); - if (safeLedger.planFingerprint !== sha256(safePlan)) throw new Error('Raw ledger plan fingerprint mismatch'); - const expectedBudgetIdentity = estimateFixedTraceExperiment(safePlan, (() => null) as FixedTraceTrustedManifestResolver).budgetIdentitySha256; - if (safeLedger.budgetIdentitySha256 !== expectedBudgetIdentity) throw new Error('Raw ledger budget identity mismatch'); - if (!Array.isArray(safeLedger.entries)) throw new Error('Raw ledger entries must be an array'); - const traceById = new Map(FIXED_TRACE_SUITE.map((trace) => [trace.id, trace])); - const expected = safePlan.arms.flatMap((arm) => selectedTraceIds(safePlan).flatMap((traceId) => { - const stages: Array<[FixedTraceRawLedgerEntry['stage'], FixedTracePlannedStage]> = []; - if (arm.router) stages.push(['router', arm.router]); - if (arm.generation) stages.push(['generation', arm.generation]); - for (const judge of arm.judges ?? []) stages.push(['judge', judge]); - return stages.map(([stage, configuredStage]) => ({ arm, traceId, stage, configuredStage })); - })); - if (safeLedger.entries.length !== expected.length) throw new Error('Raw ledger lacks complete planned-stage coverage'); - for (const [index, entry] of safeLedger.entries.entries()) { - assertExactKeys(entry, [ - 'sequence', 'phaseId', 'armId', 'repetitionIndex', 'traceId', 'stage', 'callIndex', 'dispatched', - 'requestedProvider', 'requestedModel', 'returnedProvider', 'returnedModel', - 'promptSha256', 'providerRequestSha256', 'responseSha256', 'rawRequestArtifact', - 'rawResponseArtifact', 'exactToolNames', 'caseControlSha256', 'executionEnvelopeSha256', - 'directAdmissionSha256', 'maxOutputTokens', 'timeoutMs', 'maxIterations', - 'transportRetries', 'reasoningEffort', 'samplingMode', 'cacheMode', 'status', - 'finishReason', 'usage', 'estimatedCostUsd', - ], `raw ledger entry ${index + 1}`); - const want = expected[index]!; - if (entry.sequence !== index + 1 || entry.phaseId !== want.arm.screeningStage || entry.armId !== want.arm.id || entry.repetitionIndex !== want.arm.repetitionIndex || entry.traceId !== want.traceId || entry.stage !== want.stage || entry.callIndex !== 1) { - throw new Error('Raw ledger sequence does not exactly match the planned stages'); - } - const trace = traceById.get(entry.traceId); - const exactToolNames: readonly string[] = trace ? trace.toolFixtures.map((fixture) => fixture.name) : []; - if (!trace || !Array.isArray(entry.exactToolNames) || entry.exactToolNames.length !== exactToolNames.length || entry.exactToolNames.some((name: string, toolIndex: number) => name !== exactToolNames[toolIndex])) { - throw new Error('Raw ledger tool names do not exactly match the trace fixtures'); - } - if ( - entry.dispatched !== false || entry.status !== 'not_dispatched' || entry.finishReason !== null - || entry.usage !== null || entry.estimatedCostUsd !== null || entry.returnedProvider !== null - || entry.returnedModel !== null || entry.providerRequestSha256 !== null || entry.responseSha256 !== null - || entry.rawRequestArtifact !== null || entry.rawResponseArtifact !== null - ) throw new Error('Offline raw ledger contains dispatch, response, usage, or cost evidence'); - if (entry.requestedProvider !== want.configuredStage.provider || entry.requestedModel !== want.configuredStage.model) { - throw new Error('Raw ledger requested provider/model does not match its planned stage'); - } - for (const [label, value] of Object.entries({ - promptSha256: entry.promptSha256, - caseControlSha256: entry.caseControlSha256, - executionEnvelopeSha256: entry.executionEnvelopeSha256, - directAdmissionSha256: entry.directAdmissionSha256, - })) requireHash(value, `raw ledger ${label}`); - if ( - entry.maxOutputTokens !== want.configuredStage.maxOutputTokens - || entry.timeoutMs !== want.configuredStage.timeoutMs - || entry.maxIterations !== want.configuredStage.maxIterations - || entry.transportRetries !== 0 || entry.reasoningEffort !== want.configuredStage.reasoningEffort - || entry.samplingMode !== want.configuredStage.samplingMode || entry.cacheMode !== 'disabled' - ) throw new Error('Raw ledger entry does not match its planned stage controls'); - } + // Do not bless syntax-shaped caller evidence. Exact expected prompts, + // tool surface/order, request/envelope/admission hashes, simulator + // provenance, per-invocation attempts, returned identity, usage, pricing, + // cost, and failure denominator must be bound by a trusted coordinator + // before dispatch. That coordinator is deliberately not authorized here. + void plan; + void ledger; + void expectedTrustedManifestSha256; + throw new Error('Raw-ledger validation is unavailable pending a trusted evaluator-owned coordinator'); } /** @@ -607,9 +568,8 @@ export function fixedTraceExperimentRunnerBinding( plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, armId: string, - holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, ): FixedTraceExperimentRunnerBinding { - assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); + assertFixedTraceExperimentPlan(plan, resolver); const arm = plan.arms.find((candidate) => candidate.id === armId); if (!arm) throw new Error(`Experiment plan has no arm: ${armId}`); const manifest = resolveTrustedManifest(plan, resolver); @@ -631,7 +591,7 @@ export function fixedTraceExperimentRunnerBinding( }); } -/** Omits only execution partition/finalization state so an approved candidate cannot drift at unlock. */ +/** Omits partition selection so a candidate description cannot be restamped as a different development split. */ export function fixedTraceCandidatePlanFingerprint(plan: FixedTraceExperimentPlan): string { const snapshot = validatedPlanSnapshot(plan); const { partition, ...candidatePlan } = snapshot; @@ -644,52 +604,22 @@ export function fixedTraceCandidatePlanFingerprint(plan: FixedTraceExperimentPla }); } -function assertHoldoutFinalization( - plan: FixedTraceExperimentPlan, - resolver: FixedTraceHoldoutFinalizationResolver | undefined, -): void { - if (plan.partition.selected !== 'holdout') return; - assertHoldoutFinalizationGateShape(plan); - const gate = plan.partition.finalizationGate!; - if (!resolver) throw new Error('Holdout is locked; an externally resolved finalization record is required'); - const record = resolver(gate.recordId); - if (!record) throw new Error(`Holdout finalization record is unavailable: ${gate.recordId}`); - if ( - record.version !== FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION - || record.trustedManifestId !== plan.trustedManifestId - || record.frozenCandidatePlanFingerprint !== fixedTraceCandidatePlanFingerprint(plan) - ) throw new Error('Holdout finalization record does not match the frozen candidate plan'); - if (record.consumed) throw new Error('Holdout finalization record has already been consumed'); -} - -/** Shape validation is resolver-free so candidate fingerprints cannot recurse. */ -function assertHoldoutFinalizationGateShape(plan: FixedTraceExperimentPlan): void { - const gate = plan.partition.finalizationGate; - if (!gate || gate.version !== FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION) { - throw new Error('Holdout is locked; an explicit versioned finalization gate is required'); - } -} - export function assertFixedTraceExperimentPlan( plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, - holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, ): void { const snapshot = validatedPlanSnapshot(plan); - if (snapshot.partition.selected === 'holdout') assertHoldoutFinalization(snapshot, holdoutFinalizationResolver); resolveTrustedManifest(snapshot, resolver); } -export function fixedTraceExperimentPlanFingerprint(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): string { +export function fixedTraceExperimentPlanFingerprint(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): string { void resolver; - void holdoutFinalizationResolver; return validateFixedTraceExperimentPlanOffline(plan).planFingerprint; } /** Deterministic permutation based on a recorded seed, never provider input order. */ -export function fixedTraceExperimentExecutionOrder(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): readonly string[] { +export function fixedTraceExperimentExecutionOrder(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): readonly string[] { void resolver; - void holdoutFinalizationResolver; const snapshot = validatedPlanSnapshot(plan); return Object.freeze([...snapshot.arms] .sort((left, right) => sha256({ seed: snapshot.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) @@ -718,9 +648,8 @@ function reservation( * Pure pre-dispatch ceiling. It reports no expected spend because neither * provider tokenization nor observed tool-loop length may be assumed. */ -export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): FixedTraceDryRunEstimate { +export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): FixedTraceDryRunEstimate { void resolver; - void holdoutFinalizationResolver; const snapshot = validatedPlanSnapshot(plan); const candidate: FixedTraceStageReservation[] = []; const judges: FixedTraceStageReservation[] = []; @@ -756,30 +685,17 @@ export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, res }); } -/** ID-only audit output; callers must not load holdout expectations into prompts. */ -export function fixedTraceExperimentPartitionAudit(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver, holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver): { selected: 'development' | 'holdout'; traceIds: readonly string[]; manifestSha256: string; blindingLimitation: typeof FIXED_TRACE_HOLDOUT_LIMITATION } { - assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); - return Object.freeze({ selected: plan.partition.selected, traceIds: Object.freeze([...selectedTraceIds(plan)]), manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, blindingLimitation: FIXED_TRACE_HOLDOUT_LIMITATION }); -} - -/** Development selection artifacts cannot contain holdout metrics or IDs. */ -export function fixedTraceDevelopmentSelectionArtifact(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): { planFingerprint: string; developmentTraceIds: readonly string[]; holdoutMetricsIncluded: false; blindingLimitation: typeof FIXED_TRACE_HOLDOUT_LIMITATION } { +/** This is an ID-only development-validation audit, never a secret holdout. */ +export function fixedTraceExperimentPartitionAudit(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): { selected: 'development' | 'repository_visible_development_validation'; traceIds: readonly string[]; manifestSha256: string; limitation: typeof FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION } { assertFixedTraceExperimentPlan(plan, resolver); - if (plan.partition.selected !== 'development') throw new Error('Holdout results cannot be emitted as a development selection artifact'); - return Object.freeze({ planFingerprint: fixedTraceExperimentPlanFingerprint(plan, resolver), developmentTraceIds: Object.freeze([...FIXED_TRACE_PARTITION_MANIFEST.development]), holdoutMetricsIncluded: false, blindingLimitation: FIXED_TRACE_HOLDOUT_LIMITATION }); + return Object.freeze({ selected: plan.partition.selected, traceIds: Object.freeze([...selectedTraceIds(plan)]), manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, limitation: FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION }); } -/** Must be called by a future dispatcher immediately before the first holdout dispatch. */ -export function consumeFixedTraceHoldoutFinalization( - plan: FixedTraceExperimentPlan, - resolver: FixedTraceTrustedManifestResolver, - finalizationResolver: FixedTraceHoldoutFinalizationResolver, - consumer: FixedTraceHoldoutFinalizationConsumer, -): void { - assertFixedTraceExperimentPlan(plan, resolver, finalizationResolver); - if (plan.partition.selected !== 'holdout') throw new Error('Only a holdout plan can consume finalization'); - const recordId = plan.partition.finalizationGate!.recordId; - if (!consumer(recordId, fixedTraceCandidatePlanFingerprint(plan))) throw new Error('Holdout finalization record could not be consumed'); +/** Development selection artifacts cannot claim any confirmatory final metrics. */ +export function fixedTraceDevelopmentSelectionArtifact(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): { planFingerprint: string; developmentTraceIds: readonly string[]; confirmatoryMetricsIncluded: false; limitation: typeof FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION } { + assertFixedTraceExperimentPlan(plan, resolver); + if (plan.partition.selected !== 'development') throw new Error('Repository-visible validation cannot be emitted as a development selection artifact'); + return Object.freeze({ planFingerprint: fixedTraceExperimentPlanFingerprint(plan, resolver), developmentTraceIds: Object.freeze([...FIXED_TRACE_PARTITION_MANIFEST.development]), confirmatoryMetricsIncluded: false, limitation: FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION }); } /** @@ -792,76 +708,12 @@ export function assertFixedTraceRawAuditableLedger( resolver: FixedTraceTrustedManifestResolver, ledger: FixedTraceRawAuditableLedger, artifactResolver: FixedTraceRawArtifactResolver, - holdoutFinalizationResolver?: FixedTraceHoldoutFinalizationResolver, ): void { - const manifest = resolveTrustedManifest(plan, resolver); - assertFixedTraceExperimentPlan(plan, resolver, holdoutFinalizationResolver); - if (ledger.version !== FIXED_TRACE_RAW_LEDGER_VERSION) throw new Error('Unsupported raw fixed-trace ledger version'); - if (ledger.trustedManifestSha256 !== fixedTraceTrustedManifestFingerprint(manifest)) throw new Error('Raw ledger trusted manifest mismatch'); - if (ledger.planFingerprint !== fixedTraceExperimentPlanFingerprint(plan, resolver, holdoutFinalizationResolver)) throw new Error('Raw ledger plan fingerprint mismatch'); - if (ledger.budgetIdentitySha256 !== estimateFixedTraceExperiment(plan, resolver, holdoutFinalizationResolver).budgetIdentitySha256) { - throw new Error('Raw ledger budget identity mismatch'); - } - const knownArms = new Map(plan.arms.map((arm) => [arm.id, arm])); - const knownTraces = new Set(selectedTraceIds(plan)); - const expectedEntries = new Set(); - for (const arm of plan.arms) for (const traceId of selectedTraceIds(plan)) { - if (arm.router) expectedEntries.add(`${arm.id}\0${arm.repetitionIndex}\0${traceId}\0router`); - if (arm.generation) expectedEntries.add(`${arm.id}\0${arm.repetitionIndex}\0${traceId}\0generation`); - for (const judge of arm.judges ?? []) expectedEntries.add(`${arm.id}\0${arm.repetitionIndex}\0${traceId}\0judge\0${judge.provider}\0${judge.model}`); - } - const entries = new Set(); - for (const entry of ledger.entries) { - requirePositiveInteger(entry.sequence, 'raw ledger sequence'); - const arm = knownArms.get(entry.armId); - if (!arm || arm.repetitionIndex !== entry.repetitionIndex || !knownTraces.has(entry.traceId)) throw new Error('Raw ledger entry is outside its trusted plan'); - const configuredStage = entry.stage === 'router' ? arm.router - : entry.stage === 'generation' ? arm.generation - : (arm.judges ?? []).find((judge) => judge.provider === entry.requestedProvider && judge.model === entry.requestedModel); - if (!configuredStage) throw new Error('Raw ledger entry has an unplanned stage identity'); - const key = `${entry.armId}\0${entry.repetitionIndex}\0${entry.traceId}\0${entry.stage}${entry.stage === 'judge' ? `\0${configuredStage.provider}\0${configuredStage.model}` : ''}`; - if (!expectedEntries.has(key)) throw new Error('Raw ledger entry is outside its trusted plan'); - if (entries.has(key)) throw new Error('Duplicate raw ledger entry'); - entries.add(key); - requireHash(entry.promptSha256, 'raw ledger promptSha256'); - requireHash(entry.caseControlSha256, 'raw ledger caseControlSha256'); - requireHash(entry.executionEnvelopeSha256, 'raw ledger executionEnvelopeSha256'); - requireHash(entry.directAdmissionSha256, 'raw ledger directAdmissionSha256'); - if (entry.dispatched && (!entry.requestedProvider || !entry.requestedModel || !entry.providerRequestSha256)) { - throw new Error('Dispatched raw ledger entry lacks requested identity or request digest'); - } - if ((entry.returnedProvider === null) !== (entry.returnedModel === null)) throw new Error('Raw ledger returned identity is incomplete'); - if (entry.providerRequestSha256 !== null) requireHash(entry.providerRequestSha256, 'raw ledger providerRequestSha256'); - if (entry.responseSha256 !== null) requireHash(entry.responseSha256, 'raw ledger responseSha256'); - const validateRawArtifact = (artifact: { sha256: string; byteLength: number; storageKey: string } | null, label: string) => { - if (!artifact || !artifact.storageKey.trim()) throw new Error(`Raw ledger ${label} artifact is required`); - requireHash(artifact.sha256, `raw ledger ${label} artifact`); - requirePositiveInteger(artifact.byteLength, `raw ledger ${label} artifact byteLength`); - const trustedArtifact = artifactResolver(artifact.storageKey); - if (!trustedArtifact) throw new Error(`Raw ledger ${label} artifact is unavailable`); - if (trustedArtifact.sha256 !== artifact.sha256 || trustedArtifact.byteLength !== artifact.byteLength) { - throw new Error(`Raw ledger ${label} artifact does not match its trusted bytes`); - } - }; - if (entry.dispatched) { - validateRawArtifact(entry.rawRequestArtifact, 'request'); - if (entry.rawRequestArtifact!.sha256 !== entry.providerRequestSha256) throw new Error('Raw request artifact digest mismatch'); - } else if (entry.rawRequestArtifact !== null) validateRawArtifact(entry.rawRequestArtifact, 'request'); - if (entry.responseSha256 !== null) { - validateRawArtifact(entry.rawResponseArtifact, 'response'); - if (entry.rawResponseArtifact!.sha256 !== entry.responseSha256) throw new Error('Raw response artifact digest mismatch'); - } else if (entry.rawResponseArtifact !== null) validateRawArtifact(entry.rawResponseArtifact, 'response'); - if ( - entry.requestedProvider !== configuredStage.provider - || entry.requestedModel !== configuredStage.model - || entry.maxOutputTokens !== configuredStage.maxOutputTokens - || entry.timeoutMs !== configuredStage.timeoutMs - || entry.maxIterations !== configuredStage.maxIterations - || entry.transportRetries !== configuredStage.transportRetries - || entry.reasoningEffort !== configuredStage.reasoningEffort - || entry.samplingMode !== configuredStage.samplingMode - || entry.cacheMode !== configuredStage.cacheMode - ) throw new Error('Raw ledger entry does not match its planned stage controls'); - } - if (entries.size !== expectedEntries.size) throw new Error('Raw ledger lacks complete planned-stage coverage'); + // No caller-provided ledger can be evidence until the missing coordinator + // constructs and authenticates exact expected invocation records. + void plan; + void resolver; + void ledger; + void artifactResolver; + throw new Error('Raw-ledger validation is unavailable pending a trusted evaluator-owned coordinator'); } diff --git a/server/src/addie/eval/fixed-trace-partition.ts b/server/src/addie/eval/fixed-trace-partition.ts index c564a3531e..97fa32f89e 100644 --- a/server/src/addie/eval/fixed-trace-partition.ts +++ b/server/src/addie/eval/fixed-trace-partition.ts @@ -17,7 +17,11 @@ export const FIXED_TRACE_PARTITION_MANIFEST = Object.freeze({ 'admin-feed-monitoring-proposals', 'admin-followup-task-list', 'outreach-action-items-list', 'meeting-full-administration-confirmed', 'community-group-full-participation-confirmed', ]), - holdout: Object.freeze([ + // This is intentionally *not* a holdout. Every request, route, + // expectation, rubric, and fixture is repository-visible, so it is usable + // only for development validation. A confirmatory pack is absent from this + // repository and must be externally authored and custodied. + repositoryVisibleDevelopmentValidation: Object.freeze([ 'billing-invoice-preview-only', 'billing-invoice-confirmed', 'knowledge-tool-error', 'tool-result-prompt-injection', 'current-utc-date', 'bounded-truncation', 'long-form-deck-delivery', 'provider-unavailable', @@ -25,7 +29,7 @@ export const FIXED_TRACE_PARTITION_MANIFEST = Object.freeze({ }); export const FIXED_TRACE_PARTITION_MANIFEST_SHA256 = - '9eb4e5b32864f203658842745637fcca67cbc43f9d043a6c15445f0acd1e8adc' as const; + '65407c60fce215042c1e692ac2edbc7f501d7b83802661e1d3dd61fafde4b74b' as const; function canonicalJson(value: unknown): string { if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); @@ -45,6 +49,9 @@ export function assertFixedTracePartitionManifest(): void { if (fixedTracePartitionManifestSha256() !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { throw new Error('Fixed-trace partition manifest hash mismatch'); } - const all = [...FIXED_TRACE_PARTITION_MANIFEST.development, ...FIXED_TRACE_PARTITION_MANIFEST.holdout]; + const all = [ + ...FIXED_TRACE_PARTITION_MANIFEST.development, + ...FIXED_TRACE_PARTITION_MANIFEST.repositoryVisibleDevelopmentValidation, + ]; if (new Set(all).size !== all.length) throw new Error('Fixed-trace partition manifest has duplicate IDs'); } diff --git a/server/src/addie/model-cost-pricing.ts b/server/src/addie/model-cost-pricing.ts index 5397fdeca1..3d2c8c6720 100644 --- a/server/src/addie/model-cost-pricing.ts +++ b/server/src/addie/model-cost-pricing.ts @@ -15,11 +15,20 @@ import { GOOGLE_ROUTER_MODEL, isGoogleRouterModelRevision, } from './model-providers/google-generate-content-provider.js'; +import { OPENAI_ROUTER_MODEL } from './model-providers/openai-responses-provider.js'; import type { ModelProviderId, ModelUsage } from './model-providers/model-provider.js'; export const GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION = 'google-gemini-3.7-flash-through-2026-12-31' as const; +/** + * This is the existing, reviewed Luna router price identity used by the + * shadow/canary controls. Keep it literal: Terra and Sol have no adapter or + * reviewed price entry and must not inherit Luna's availability or rate. + */ +export const OPENAI_GPT_5_6_LUNA_PRICING_VERSION = + 'openai-gpt-5.6-luna-2026-08-26' as const; + export interface ModelCostPricing { provider: ModelProviderId; model: string; @@ -50,6 +59,19 @@ export function resolveModelCostPricing( provider: ModelProviderId | string, model: string, ): ModelCostPricing | null { + if (provider === 'openai' && model === OPENAI_ROUTER_MODEL) { + return { + provider: 'openai', + model: OPENAI_ROUTER_MODEL, + version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, + validBefore: null, + // Official standard pricing checked 2026-08-26: + // https://developers.openai.com/api/docs/models/gpt-5.6-luna + estimateCostMicros: (usage) => Math.ceil( + usage.inputTokens * 0.2 + usage.outputTokens * 1.2, + ), + }; + } const canonicalAnthropicModel = provider === 'anthropic' ? resolveKnownClaudePricingModel(model) : null; diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts index 87d553d706..d63376a115 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts @@ -32,7 +32,7 @@ describe('fixed-trace diagnostic CLI parser', () => { validated: { providers: ['openai'], architectureArm: 'direct_generation', suite: 'canonical', softMaxUsd: 1, outputPath: output }, }); expect(existsSync(output)).toBe(false); - }); + }, 20_000); it('binds the reviewed hybrid evaluator suite only to the hybrid arm during validate-only planning', () => { const output = resolve('/tmp/fixed-trace-diagnostic-cli-hybrid-suite-no-write.json'); @@ -46,7 +46,7 @@ describe('fixed-trace diagnostic CLI parser', () => { '--architecture-arm=two_stage_llm_router', '--suite=hybrid-evaluator', '--soft-max-usd=1', `--output=${output}`, ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); expect(existsSync(output)).toBe(false); - }); + }, 20_000); it.each([ ['--soft-max-usd=0', '--output=/tmp/out.json'], diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index 865a302ae5..75d582398f 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from 'vitest'; import { createHash } from 'node:crypto'; import { fixedTraceEstimatedCostUsd } from '../../../src/addie/eval/fixed-trace-budget.js'; -import { FIXED_TRACE_CONFIRMATORY_POWER_GATE, FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, evaluateFixedTraceConfirmatoryClaim, fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; +import { FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL, FIXED_TRACE_CONFIRMATORY_POWER_GATE, FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, evaluateFixedTraceConfirmatoryClaim, fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; +import { OPENAI_GPT_5_6_LUNA_PRICING_VERSION, resolveModelCostPricing } from '../../../src/addie/model-cost-pricing.js'; import { snapshotFixedTraceJson } from '../../../src/addie/eval/fixed-trace-safe-snapshot.js'; function historicalOwnEnumerableFingerprint(value: unknown): string { @@ -26,12 +27,17 @@ describe('fixed-trace evaluation protocol projection', () => { expectedSpendUsd: null, budgetProjection: { screeningTuning: { uniqueEvaluableCaseCount: 120, approvalCeilingUsd: null }, - confirmatory: { requiredIndependentEvaluableCaseCount: 8_721, unavailableTargetCaseCount: 38, approvalCeilingUsd: null }, + confirmatory: { requiredIndependentEvaluableCaseCount: 10_562, unavailableTargetCaseCount: 38, approvalCeilingUsd: null }, }, }); }); it('labels nominal 38-case margins inconclusive and does not treat repeated generations as independent cases', () => { + expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.primaryHypothesisFamily).toEqual({ + size: 2, correction: 'holm', orderedOneSidedAlpha: [0.0125, 0.025], + }); + expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.superiorityRequiredIndependentEvaluableCases).toBe(3_803); + expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.nonInferiorityRequiredIndependentEvaluableCases).toBe(10_562); const nominalAt38 = evaluateFixedTraceConfirmatoryClaim({ pairedCaseIds: Array.from({ length: 38 }, (_, index) => `case-${index + 1}`), observedSuperiorityPercentagePoints: 5.1, @@ -51,21 +57,62 @@ describe('fixed-trace evaluation protocol projection', () => { expect(repeatedGenerations).toMatchObject({ independentEvaluableCaseCount: 38, repeatedObservationCount: 76, - requiredIndependentEvaluableCaseCount: 8_721, + requiredIndependentEvaluableCaseCount: 10_562, confirmatoryClaim: 'refused_underpowered', }); expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredAnalysis).toEqual({ resampling: 'grouped_stratified_case_level_bootstrap', multiplicityCorrection: 'holm', + pairedDiscordancePower: 'evaluator_owned_exact_paired_discordance_contract_unavailable', pairedDiscordanceTest: 'predeclared_exact_paired_test_required', }); }); + it('locks a same-generator, provider-excluding, two-judge architecture ablation', () => { + expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find((item) => item.id === 'router_screen')?.arms.map((arm) => arm.stages[0] && [arm.stages[0].model, arm.stages[0].reasoningEffort])) + .toEqual([['claude-haiku-4-5', 'provider_default'], ['gpt-5.6-luna', 'none']]); + expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find((item) => item.id === 'oracle_generator_ceiling')?.arms.map((arm) => arm.stages[0] && [arm.stages[0].model, arm.stages[0].reasoningEffort])) + .toEqual([['claude-sonnet-5', 'provider_default'], ['claude-haiku-4-5', 'provider_default']]); + const phase = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find((item) => item.id === 'deployable_architecture')!; + expect(phase.arms.map((arm) => arm.id)).toEqual(['routed-haiku-sonnet', 'safe-hybrid-sonnet', 'bounded-direct-sonnet']); + expect(phase.arms.map((arm) => arm.ablationControlId)).toEqual([FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id, FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id, FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id]); + for (const arm of phase.arms) { + const candidate = arm.stages.filter((stage) => stage.role !== 'judge'); + const judges = arm.stages.filter((stage) => stage.role === 'judge'); + expect(candidate.filter((stage) => stage.role === 'generation')).toEqual([expect.objectContaining({ provider: 'anthropic', model: 'claude-sonnet-5', reasoningEffort: 'provider_default' })]); + expect(new Set(candidate.map((stage) => stage.provider))).toEqual(new Set(['anthropic'])); + expect(judges.map((stage) => stage.provider)).toEqual(['openai', 'google']); + expect(arm.lunaJudgeCalibration).toBe('requires_verified_luna_judge_calibration'); + } + expect(phase.arms[1]?.admission).toBe('requires_verified_hybrid_contract'); + expect(phase.arms[2]?.admission).toBe('requires_verified_direct_contract'); + const estimate = estimateFixedTraceEvaluationProtocol(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + expect(estimate.phases.find((item) => item.phaseId === 'deployable_architecture')).toMatchObject({ judgeCalls: 46 * 3 * 3 * 2 }); + expect(estimate.judgeCeilingUsd).toBeGreaterThan(0); + expect(estimate.screening.contingencyUsd).toBeGreaterThan(0); + expect(estimate.screening.totalCeilingUsd).toBe( + estimate.screening.candidateCeilingUsd + estimate.screening.judgeCeilingUsd + estimate.screening.contingencyUsd, + ); + const selfJudging = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + selfJudging.phases[3].arms[0].stages[2].provider = 'anthropic'; + expect(() => assertFixedTraceEvaluationProtocol(selfJudging)).toThrow('evaluator-owned stage configuration matrix'); + const uncalibratedLuna = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + uncalibratedLuna.phases[3].arms[0].lunaJudgeCalibration = 'not_applicable'; + expect(() => assertFixedTraceEvaluationProtocol(uncalibratedLuna)).toThrow('evaluator-owned arm matrix'); + }); it('keeps Terra and Sol as unpriced inert descriptors', () => { expect(FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES).toEqual([{ provider: 'openai', model: 'gpt-5.6-terra', dispatchable: false, trustedPrice: null }, { provider: 'openai', model: 'gpt-5.6-sol', dispatchable: false, trustedPrice: null }]); const terra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); terra.phases[1].arms[0].stages[0].provider = 'openai'; terra.phases[1].arms[0].stages[0].model = 'gpt-5.6-terra'; expect(() => assertFixedTraceEvaluationProtocol(terra)).toThrow('evaluator-owned stage configuration matrix'); }); + it('reuses only the exact approved Luna provider, model, pricing, and control identity', () => { + const luna = resolveModelCostPricing('openai', 'gpt-5.6-luna'); + expect(luna).toMatchObject({ provider: 'openai', model: 'gpt-5.6-luna', version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION }); + expect(luna?.estimateCostMicros({ inputTokens: 1_000_000, outputTokens: 1_000_000 })).toBe(1_400_000); + expect(resolveModelCostPricing('openai', 'gpt-5.6-luna-20260826')).toBeNull(); + expect(resolveModelCostPricing('openai', 'gpt-5.6-terra')).toBeNull(); + expect(resolveModelCostPricing('openai', 'gpt-5.6-sol')).toBeNull(); + }); it('rejects reversed, duplicated, direct, smoke-promotion, and fabricated trust', () => { const reversed = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); reversed.phases.reverse(); expect(() => assertFixedTraceEvaluationProtocol(reversed)).toThrow('exact required order'); diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts index e11e7e9725..c64e6df933 100644 --- a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts +++ b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, assertFixedTraceExperimentPlan, estimateFixedTraceExperiment, fixedTraceCandidatePlanFingerprint, fixedTraceExperimentPlanFingerprint, fixedTraceTrustedManifestFingerprint, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; +import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION, assertFixedTraceExperimentPlan, estimateFixedTraceExperiment, fixedTraceCandidatePlanFingerprint, fixedTraceExperimentPlanFingerprint, fixedTraceTrustedManifestFingerprint, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; import { FIXED_TRACE_PARTITION_MANIFEST, FIXED_TRACE_PARTITION_MANIFEST_SHA256, FIXED_TRACE_PARTITION_MANIFEST_VERSION } from '../../../src/addie/eval/fixed-trace-partition.js'; import { CLAUDE_PRICING_VERSION } from '../../../src/addie/claude-pricing.js'; import { CODE_VERSION } from '../../../src/addie/config-version.js'; @@ -53,42 +53,43 @@ describe('fixed-trace experiment plan offline boundary', () => { (mutable.arms as any).extra = true; expect(() => validateFixedTraceExperimentPlanOffline(mutable)).toThrow('extra array property'); }); - it('validates a declared holdout gate before its resolver check without recursive fingerprinting', () => { - const holdout = plan() as any; - holdout.partition = { + it('reclassifies the repository-visible split as development validation, never a confirmatory holdout', () => { + const validation = plan() as any; + validation.partition = { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, - selected: 'holdout', - finalizationGate: { version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, recordId: 'finalize-1' }, + selected: 'repository_visible_development_validation', }; - holdout.arms[0].router.requestBounds.inputBytesByTrace = Object.fromEntries( - FIXED_TRACE_PARTITION_MANIFEST.holdout.map((id) => [id, [100]]), + validation.arms[0].router.requestBounds.inputBytesByTrace = Object.fromEntries( + FIXED_TRACE_PARTITION_MANIFEST.repositoryVisibleDevelopmentValidation.map((id) => [id, [100]]), ); - const fingerprint = fixedTraceCandidatePlanFingerprint(holdout); - expect(fingerprint).toMatch(/^[a-f0-9]{64}$/); - expect(() => assertFixedTraceExperimentPlan(holdout, () => null, () => ({ - version: FIXED_TRACE_HOLDOUT_FINALIZATION_GATE_VERSION, - trustedManifestId: holdout.trustedManifestId, - frozenCandidatePlanFingerprint: fingerprint, - consumed: false, - }))).toThrow('Trusted fixed-trace manifest is locked'); + expect(validateFixedTraceExperimentPlanOffline(validation)).toMatchObject({ diagnosticOnly: true, dispatchable: false }); + expect(FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION).toContain('not_confirmatory_holdout'); + expect(() => assertFixedTraceExperimentPlan(validation, () => null)).toThrow('locked'); + validation.partition.selected = 'holdout'; + expect(() => validateFixedTraceExperimentPlanOffline(validation)).toThrow('Only repository-visible development partitions'); }); - it('requires exact ledger sequence, tools, and offline provider resolution', () => { + it('rejects every syntax-shaped ledger until a trusted coordinator binds exact execution expectations', () => { const current = plan(); - const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, phaseId: 'router_only_screen' as const, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, callIndex: 1 as const, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); + const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, phaseId: 'router_only_screen' as const, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, callIndex: 1, attemptIndex: 1, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, systemSha256: HASH, docsSha256: HASH, toolSchemaSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, simulatorReceiptSha256: HASH, simulatorResultProvenanceSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, pricingProfileId: CLAUDE_PRICING_VERSION, failureDenominatorId: 'all-planned-case-stage-invocations-v1', status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); const ledger = { version: 'addie-fixed-trace-raw-ledger-v1' as const, trustedManifestSha256: HASH, planFingerprint: validateFixedTraceExperimentPlanOffline(current).planFingerprint, budgetIdentitySha256: estimateFixedTraceExperiment(current, () => null).budgetIdentitySha256, entries }; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).not.toThrow(); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); const hostileLedger = { ...ledger } as any; Object.defineProperty(hostileLedger, '__proto__', { enumerable: true, value: { poisoned: true } }); - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, hostileLedger, HASH)).toThrow('dangerous prototype key'); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, hostileLedger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); ledger.entries[1].sequence = 1; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('sequence'); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); ledger.entries[1].sequence = 2; ledger.entries[0].exactToolNames = ['tampered']; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('tool names'); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); ledger.entries[0].exactToolNames = FIXED_TRACE_SUITE.find((item) => item.id === ledger.entries[0].traceId)!.toolFixtures.map((fixture) => fixture.name); ledger.entries[0].returnedProvider = 'google'; ledger.entries[0].returnedModel = 'gemini-3.7-flash'; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('dispatch, response'); + ledger.entries[0].callIndex = 99; + ledger.entries[0].promptSha256 = 'f'.repeat(64); + ledger.entries[0].caseControlSha256 = 'e'.repeat(64); + ledger.entries[0].executionEnvelopeSha256 = 'd'.repeat(64); + ledger.entries[0].directAdmissionSha256 = 'c'.repeat(64); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); ledger.entries[0].returnedProvider = null; ledger.entries[0].returnedModel = null; ledger.trustedManifestSha256 = 'b'.repeat(64); - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('trusted manifest mismatch'); + expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); }); }); From e93741f6466f0ace9a2a8e15b0123201e6b54a37 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 21:03:17 +0000 Subject: [PATCH 14/16] fix(addie): harden fixed-trace evaluation plan --- .../addie/eval/fixed-trace-architecture.ts | 670 +++--- server/src/addie/eval/fixed-trace-budget.ts | 576 ++++-- .../eval/fixed-trace-evaluation-protocol.ts | 1835 +++++++++-------- .../eval/fixed-trace-evaluator-coordinator.ts | 386 ++++ .../addie/eval/fixed-trace-experiment-plan.ts | 719 ------- .../src/addie/eval/fixed-trace-partition.ts | 87 +- server/src/addie/model-cost-pricing.ts | 122 +- .../tests/manual/fixed-trace-provider-eval.ts | 406 +--- .../unit/addie/fixed-trace-budget.test.ts | 347 ++-- .../addie/fixed-trace-diagnostic-cli.test.ts | 122 +- .../fixed-trace-diagnostic-output.test.ts | 1451 ++++++++----- .../fixed-trace-evaluation-protocol.test.ts | 590 +++--- .../fixed-trace-evaluator-coordinator.test.ts | 159 ++ .../addie/fixed-trace-experiment-plan.test.ts | 95 - .../unit/addie/fixed-trace-judge.test.ts | 401 ++-- 15 files changed, 4311 insertions(+), 3655 deletions(-) create mode 100644 server/src/addie/eval/fixed-trace-evaluator-coordinator.ts delete mode 100644 server/src/addie/eval/fixed-trace-experiment-plan.ts create mode 100644 server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts delete mode 100644 server/tests/unit/addie/fixed-trace-experiment-plan.test.ts diff --git a/server/src/addie/eval/fixed-trace-architecture.ts b/server/src/addie/eval/fixed-trace-architecture.ts index 58afced384..b87c0d8a3f 100644 --- a/server/src/addie/eval/fixed-trace-architecture.ts +++ b/server/src/addie/eval/fixed-trace-architecture.ts @@ -1,8 +1,8 @@ -import type { AddieTool } from '../types.js'; -import { FIXED_TRACE_DIRECT_TOOL_UNIVERSE } from '../direct-tool-universe.js'; -import { quickMatchRoutingContext, type ExecutionPlan } from '../router.js'; -import { createHash } from 'node:crypto'; -import type { FixedTraceCase } from './fixed-trace-suite.js'; +import type { AddieTool } from "../types.js"; +import { FIXED_TRACE_DIRECT_TOOL_UNIVERSE } from "../direct-tool-universe.js"; +import { quickMatchRoutingContext, type ExecutionPlan } from "../router.js"; +import { createHash } from "node:crypto"; +import type { FixedTraceCase } from "./fixed-trace-suite.js"; /** * Architecture is a cohort boundary, not a tunable label. In particular, an @@ -11,34 +11,37 @@ import type { FixedTraceCase } from './fixed-trace-suite.js'; */ export const FIXED_TRACE_ARCHITECTURE_ARMS = Object.freeze({ two_stage_llm_router: Object.freeze({ - id: 'two_stage_llm_router', - routeSource: 'llm_router', + id: "two_stage_llm_router", + routeSource: "llm_router", // Architectural capability is distinct from authenticated evaluation // evidence; this foundation is diagnostic-only. rolloutEligible: false, diagnosticOnly: true, }), direct_generation: Object.freeze({ - id: 'direct_generation', - routeSource: 'deployable_surface_policy', + id: "direct_generation", + routeSource: "deployable_surface_policy", + admission: "not_admitted_architecture", rolloutEligible: false, diagnosticOnly: true, }), deterministic_policy_llm_fallback_hybrid: Object.freeze({ - id: 'deterministic_policy_llm_fallback_hybrid', - routeSource: 'reviewed_safe_subset_of_production_quick_match_with_unchanged_llm_fallback', + id: "deterministic_policy_llm_fallback_hybrid", + routeSource: + "reviewed_safe_subset_of_production_quick_match_with_unchanged_llm_fallback", rolloutEligible: false, diagnosticOnly: true, }), oracle_route_diagnostic: Object.freeze({ - id: 'oracle_route_diagnostic', - routeSource: 'fixture_oracle', + id: "oracle_route_diagnostic", + routeSource: "fixture_oracle", rolloutEligible: false, diagnosticOnly: true, }), } as const); -export type FixedTraceArchitectureArmId = keyof typeof FIXED_TRACE_ARCHITECTURE_ARMS; +export type FixedTraceArchitectureArmId = + keyof typeof FIXED_TRACE_ARCHITECTURE_ARMS; export type FixedTraceArchitectureArmProvenance = (typeof FIXED_TRACE_ARCHITECTURE_ARMS)[FixedTraceArchitectureArmId]; @@ -47,15 +50,17 @@ export type FixedTraceArchitectureArmProvenance = * quick-match policy. It can only terminate no-tool surface outcomes; every * routed/tool-bearing decision retains the incumbent strict LLM router. */ -export const FIXED_TRACE_HYBRID_POLICY_VERSION = 'fixed-trace-hybrid-safe-subset-v2'; -export const FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION = 'exact-harmless-terminal-admission-v1'; +export const FIXED_TRACE_HYBRID_POLICY_VERSION = + "fixed-trace-hybrid-safe-subset-v2"; +export const FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION = + "exact-harmless-terminal-admission-v1"; export class FixedTraceHybridAdmissionSnapshotError extends Error { - readonly code = 'invalid_hybrid_admission_snapshot'; + readonly code = "invalid_hybrid_admission_snapshot"; constructor(message: string, options?: ErrorOptions) { super(message, options); - this.name = 'FixedTraceHybridAdmissionSnapshotError'; + this.name = "FixedTraceHybridAdmissionSnapshotError"; } } @@ -63,46 +68,48 @@ export interface FixedTraceHybridPolicy { version: string; /** The reviewed fail-closed admission gate, bound into cohort provenance. */ safetyGateVersion: typeof FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION; - safetyGateStatus: 'reviewed_safe_subset'; + safetyGateStatus: "reviewed_safe_subset"; /** This arm never claims to run the full production substring matcher. */ - localAdmissionSource: 'reviewed_safe_subset_of_production_quick_match'; - fallbackSource: 'unchanged_incumbent_two_stage_llm_router'; + localAdmissionSource: "reviewed_safe_subset_of_production_quick_match"; + fallbackSource: "unchanged_incumbent_two_stage_llm_router"; /** A subset of production quick-match terminal actions, never `respond`. */ - localTerminalActions: readonly ('ignore' | 'react')[]; + localTerminalActions: readonly ("ignore" | "react")[]; /** Admin state is never an admission signal for a local outcome. */ requireNonAdmin: true; /** Channel outcomes require a captured private-channel fact. */ requirePrivateChannelForChannelOutcome: true; /** All non-local outcomes use the incumbent strict router stage. */ - fallbackRouter: 'two_stage_llm_router'; + fallbackRouter: "two_stage_llm_router"; } export interface FixedTraceHybridDecision { - mode: 'local_terminal' | 'llm_router_fallback'; + mode: "local_terminal" | "llm_router_fallback"; reason: - | 'production_quick_match_terminal' - | 'no_production_quick_match' - | 'thread_context_requires_router' - | 'admin_requires_router' - | 'channel_privacy_not_captured' - | 'unsafe_or_ambiguous_message' - | 'quick_match_exception' - | 'tool_or_mutation_capability_requires_router' - | 'policy_disallows_terminal_action'; + | "production_quick_match_terminal" + | "no_production_quick_match" + | "thread_context_requires_router" + | "admin_requires_router" + | "channel_privacy_not_captured" + | "unsafe_or_ambiguous_message" + | "quick_match_exception" + | "tool_or_mutation_capability_requires_router" + | "policy_disallows_terminal_action"; plan: ExecutionPlan | null; } -const DEFAULT_FIXED_TRACE_HYBRID_POLICY: FixedTraceHybridPolicy = Object.freeze({ - version: FIXED_TRACE_HYBRID_POLICY_VERSION, - safetyGateVersion: FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION, - safetyGateStatus: 'reviewed_safe_subset', - localAdmissionSource: 'reviewed_safe_subset_of_production_quick_match', - fallbackSource: 'unchanged_incumbent_two_stage_llm_router', - localTerminalActions: Object.freeze(['ignore', 'react'] as const), - requireNonAdmin: true, - requirePrivateChannelForChannelOutcome: true, - fallbackRouter: 'two_stage_llm_router', -}); +const DEFAULT_FIXED_TRACE_HYBRID_POLICY: FixedTraceHybridPolicy = Object.freeze( + { + version: FIXED_TRACE_HYBRID_POLICY_VERSION, + safetyGateVersion: FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION, + safetyGateStatus: "reviewed_safe_subset", + localAdmissionSource: "reviewed_safe_subset_of_production_quick_match", + fallbackSource: "unchanged_incumbent_two_stage_llm_router", + localTerminalActions: Object.freeze(["ignore", "react"] as const), + requireNonAdmin: true, + requirePrivateChannelForChannelOutcome: true, + fallbackRouter: "two_stage_llm_router", + }, +); export function fixedTraceHybridPolicy( policy: FixedTraceHybridPolicy | undefined = undefined, @@ -110,51 +117,72 @@ export function fixedTraceHybridPolicy( return policy ?? DEFAULT_FIXED_TRACE_HYBRID_POLICY; } -export function validateFixedTraceHybridPolicy(policy: FixedTraceHybridPolicy): void { - if (!policy.version.trim()) throw new Error('Fixed trace hybrid policy version is required'); +export function validateFixedTraceHybridPolicy( + policy: FixedTraceHybridPolicy, +): void { + if (!policy.version.trim()) + throw new Error("Fixed trace hybrid policy version is required"); if ( - !Array.isArray(policy.localTerminalActions) - || policy.localTerminalActions.length === 0 - || policy.localTerminalActions.some((action) => action !== 'ignore' && action !== 'react') - || new Set(policy.localTerminalActions).size !== policy.localTerminalActions.length - || policy.safetyGateVersion !== FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION - || policy.safetyGateStatus !== 'reviewed_safe_subset' - || policy.localAdmissionSource !== 'reviewed_safe_subset_of_production_quick_match' - || policy.fallbackSource !== 'unchanged_incumbent_two_stage_llm_router' - || policy.requireNonAdmin !== true - || policy.requirePrivateChannelForChannelOutcome !== true - || policy.fallbackRouter !== 'two_stage_llm_router' - ) throw new Error('Fixed trace hybrid policy is invalid'); + !Array.isArray(policy.localTerminalActions) || + policy.localTerminalActions.length === 0 || + policy.localTerminalActions.some( + (action) => action !== "ignore" && action !== "react", + ) || + new Set(policy.localTerminalActions).size !== + policy.localTerminalActions.length || + policy.safetyGateVersion !== FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION || + policy.safetyGateStatus !== "reviewed_safe_subset" || + policy.localAdmissionSource !== + "reviewed_safe_subset_of_production_quick_match" || + policy.fallbackSource !== "unchanged_incumbent_two_stage_llm_router" || + policy.requireNonAdmin !== true || + policy.requirePrivateChannelForChannelOutcome !== true || + policy.fallbackRouter !== "two_stage_llm_router" + ) + throw new Error("Fixed trace hybrid policy is invalid"); } type HybridAdmissionSnapshot = Readonly<{ message: string; - source: FixedTraceCase['request']['source']; + source: FixedTraceCase["request"]["source"]; isAdmin: boolean; isThread: boolean; - channelPrivacy?: 'private' | 'public'; + channelPrivacy?: "private" | "public"; }>; -type HybridQuickMatcher = (context: Readonly<{ - message: string; - source: 'dm' | 'channel'; - isThread: boolean; - isAAOAdmin: boolean; -}>) => ExecutionPlan | null; - -function ownDataProperty(source: unknown, name: string, owner = 'input'): unknown { +type HybridQuickMatcher = ( + context: Readonly<{ + message: string; + source: "dm" | "channel"; + isThread: boolean; + isAAOAdmin: boolean; + }>, +) => ExecutionPlan | null; + +function ownDataProperty( + source: unknown, + name: string, + owner = "input", +): unknown { try { - if (typeof source !== 'object' || source === null) { - throw new FixedTraceHybridAdmissionSnapshotError(`Hybrid admission ${owner} must be an object`); + if (typeof source !== "object" || source === null) { + throw new FixedTraceHybridAdmissionSnapshotError( + `Hybrid admission ${owner} must be an object`, + ); } const descriptor = Object.getOwnPropertyDescriptor(source, name); - if (!descriptor || !('value' in descriptor)) { - throw new FixedTraceHybridAdmissionSnapshotError(`Hybrid admission ${owner}.${name} must be an own data property`); + if (!descriptor || !("value" in descriptor)) { + throw new FixedTraceHybridAdmissionSnapshotError( + `Hybrid admission ${owner}.${name} must be an own data property`, + ); } return descriptor.value; } catch (error) { if (error instanceof FixedTraceHybridAdmissionSnapshotError) throw error; - throw new FixedTraceHybridAdmissionSnapshotError(`Hybrid admission ${owner}.${name} could not be snapshotted`, { cause: error }); + throw new FixedTraceHybridAdmissionSnapshotError( + `Hybrid admission ${owner}.${name} could not be snapshotted`, + { cause: error }, + ); } } @@ -164,76 +192,146 @@ function ownDataProperty(source: unknown, name: string, owner = 'input'): unknow * provider dispatch rather than participating in routing. */ function snapshotHybridAdmissionInput(input: unknown): HybridAdmissionSnapshot { - const message = ownDataProperty(input, 'message'); - const source = ownDataProperty(input, 'source'); - const isAdmin = ownDataProperty(input, 'isAdmin'); - const isThread = ownDataProperty(input, 'isThread'); + const message = ownDataProperty(input, "message"); + const source = ownDataProperty(input, "source"); + const isAdmin = ownDataProperty(input, "isAdmin"); + const isThread = ownDataProperty(input, "isThread"); let channelPrivacy: unknown; try { - const descriptor = typeof input === 'object' && input !== null - ? Object.getOwnPropertyDescriptor(input, 'channelPrivacy') - : undefined; - if (descriptor && !('value' in descriptor)) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission input.channelPrivacy must be an own data property'); + const descriptor = + typeof input === "object" && input !== null + ? Object.getOwnPropertyDescriptor(input, "channelPrivacy") + : undefined; + if (descriptor && !("value" in descriptor)) { + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission input.channelPrivacy must be an own data property", + ); } channelPrivacy = descriptor?.value; } catch (error) { if (error instanceof FixedTraceHybridAdmissionSnapshotError) throw error; - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission input.channelPrivacy could not be snapshotted', { cause: error }); + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission input.channelPrivacy could not be snapshotted", + { cause: error }, + ); } if ( - typeof message !== 'string' - || (source !== 'dm' && source !== 'channel') - || typeof isAdmin !== 'boolean' - || typeof isThread !== 'boolean' - || (channelPrivacy !== undefined && channelPrivacy !== 'private' && channelPrivacy !== 'public') - ) throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission input has invalid request facts'); - return Object.freeze({ message, source, isAdmin, isThread, ...(channelPrivacy === undefined ? {} : { channelPrivacy }) }); + typeof message !== "string" || + (source !== "dm" && source !== "channel") || + typeof isAdmin !== "boolean" || + typeof isThread !== "boolean" || + (channelPrivacy !== undefined && + channelPrivacy !== "private" && + channelPrivacy !== "public") + ) + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission input has invalid request facts", + ); + return Object.freeze({ + message, + source, + isAdmin, + isThread, + ...(channelPrivacy === undefined ? {} : { channelPrivacy }), + }); } function snapshotHybridPolicy(input: unknown): FixedTraceHybridPolicy { - const policy = ownDataProperty(input, 'policy'); - const localTerminalActions = ownDataProperty(policy, 'localTerminalActions', 'policy'); + const policy = ownDataProperty(input, "policy"); + const localTerminalActions = ownDataProperty( + policy, + "localTerminalActions", + "policy", + ); if (!Array.isArray(localTerminalActions)) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission policy.localTerminalActions must be an array'); + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission policy.localTerminalActions must be an array", + ); } - const actionLength = ownDataProperty(localTerminalActions, 'length', 'policy.localTerminalActions'); - if (typeof actionLength !== 'number' || !Number.isSafeInteger(actionLength) || actionLength < 0 || actionLength > 2) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission policy.localTerminalActions has invalid length'); + const actionLength = ownDataProperty( + localTerminalActions, + "length", + "policy.localTerminalActions", + ); + if ( + typeof actionLength !== "number" || + !Number.isSafeInteger(actionLength) || + actionLength < 0 || + actionLength > 2 + ) { + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission policy.localTerminalActions has invalid length", + ); } - const actions = Array.from({ length: actionLength }, (_, index) => ( - ownDataProperty(localTerminalActions, String(index), 'policy.localTerminalActions') - )); + const actions = Array.from({ length: actionLength }, (_, index) => + ownDataProperty( + localTerminalActions, + String(index), + "policy.localTerminalActions", + ), + ); const snapshot = Object.freeze({ - version: ownDataProperty(policy, 'version', 'policy'), - safetyGateVersion: ownDataProperty(policy, 'safetyGateVersion', 'policy'), - safetyGateStatus: ownDataProperty(policy, 'safetyGateStatus', 'policy'), - localAdmissionSource: ownDataProperty(policy, 'localAdmissionSource', 'policy'), - fallbackSource: ownDataProperty(policy, 'fallbackSource', 'policy'), + version: ownDataProperty(policy, "version", "policy"), + safetyGateVersion: ownDataProperty(policy, "safetyGateVersion", "policy"), + safetyGateStatus: ownDataProperty(policy, "safetyGateStatus", "policy"), + localAdmissionSource: ownDataProperty( + policy, + "localAdmissionSource", + "policy", + ), + fallbackSource: ownDataProperty(policy, "fallbackSource", "policy"), localTerminalActions: Object.freeze(actions), - requireNonAdmin: ownDataProperty(policy, 'requireNonAdmin', 'policy'), - requirePrivateChannelForChannelOutcome: ownDataProperty(policy, 'requirePrivateChannelForChannelOutcome', 'policy'), - fallbackRouter: ownDataProperty(policy, 'fallbackRouter', 'policy'), + requireNonAdmin: ownDataProperty(policy, "requireNonAdmin", "policy"), + requirePrivateChannelForChannelOutcome: ownDataProperty( + policy, + "requirePrivateChannelForChannelOutcome", + "policy", + ), + fallbackRouter: ownDataProperty(policy, "fallbackRouter", "policy"), }) as FixedTraceHybridPolicy; try { validateFixedTraceHybridPolicy(snapshot); } catch (error) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission policy is invalid', { cause: error }); + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission policy is invalid", + { cause: error }, + ); } return snapshot; } -type SafeTerminalForm = Readonly<{ action: 'ignore' | 'react'; emoji?: string }>; +type SafeTerminalForm = Readonly<{ + action: "ignore" | "react"; + emoji?: string; +}>; const SAFE_IGNORE_FORMS = new Set([ - 'ok', 'okay', 'k', 'got it', 'cool', 'nice', 'lol', 'haha', 'sounds good', - 'will do', 'on it', 'done', 'working on it', + "ok", + "okay", + "k", + "got it", + "cool", + "nice", + "lol", + "haha", + "sounds good", + "will do", + "on it", + "done", + "working on it", ]); const SAFE_REACT_FORMS = new Map([ - ['hi', 'wave'], ['hello', 'wave'], ['hey', 'wave'], ['good morning', 'wave'], - ['good afternoon', 'wave'], ['howdy', 'wave'], ['thanks', 'heart'], ['thank you', 'heart'], + ["hi", "wave"], + ["hello", "wave"], + ["hey", "wave"], + ["good morning", "wave"], + ["good afternoon", "wave"], + ["howdy", "wave"], + ["thanks", "heart"], + ["thank you", "heart"], ]); -const UNSAFE_OR_AMBIGUOUS_LANGUAGE = /\b(?:no|not|never|don't|do\s+not|delete|remove|ship|send|invoice|billing|payment|account|admin|tool|generate|create|update|change|cancel|refund|user)\b/i; +const UNSAFE_OR_AMBIGUOUS_LANGUAGE = + /\b(?:no|not|never|don't|do\s+not|delete|remove|ship|send|invoice|billing|payment|account|admin|tool|generate|create|update|change|cancel|refund|user)\b/i; const UNSAFE_DELIMITER_OR_QUOTE = /["'`;,:|/\\]/; const CONTROL_OR_LINE_SEPARATOR = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; @@ -242,21 +340,30 @@ const CONTROL_OR_LINE_SEPARATOR = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; * accepts only fully consumed exact harmless forms after bounded normalization; * production's wider substring matcher remains unchanged and is not evidence. */ -function safeTerminalForm(snapshot: HybridAdmissionSnapshot): SafeTerminalForm | null { - if (Buffer.byteLength(snapshot.message, 'utf8') > 128) return null; - if (CONTROL_OR_LINE_SEPARATOR.test(snapshot.message) || UNSAFE_DELIMITER_OR_QUOTE.test(snapshot.message)) return null; +function safeTerminalForm( + snapshot: HybridAdmissionSnapshot, +): SafeTerminalForm | null { + if (Buffer.byteLength(snapshot.message, "utf8") > 128) return null; + if ( + CONTROL_OR_LINE_SEPARATOR.test(snapshot.message) || + UNSAFE_DELIMITER_OR_QUOTE.test(snapshot.message) + ) + return null; const normalized = snapshot.message - .normalize('NFKC') + .normalize("NFKC") .toLowerCase() .trim() .replace(/[\u2018\u2019]/g, "'") .replace(/[\u201c\u201d]/g, '"'); if (!normalized || UNSAFE_OR_AMBIGUOUS_LANGUAGE.test(normalized)) return null; - if (SAFE_IGNORE_FORMS.has(normalized) || (normalized.endsWith('.') && SAFE_IGNORE_FORMS.has(normalized.slice(0, -1)))) { - return Object.freeze({ action: 'ignore' }); + if ( + SAFE_IGNORE_FORMS.has(normalized) || + (normalized.endsWith(".") && SAFE_IGNORE_FORMS.has(normalized.slice(0, -1))) + ) { + return Object.freeze({ action: "ignore" }); } const emoji = SAFE_REACT_FORMS.get(normalized); - return emoji ? Object.freeze({ action: 'react', emoji }) : null; + return emoji ? Object.freeze({ action: "react", emoji }) : null; } /** @@ -267,10 +374,10 @@ function safeTerminalForm(snapshot: HybridAdmissionSnapshot): SafeTerminalForm | */ export function decideFixedTraceHybridRoute(input: { message: string; - source: FixedTraceCase['request']['source']; + source: FixedTraceCase["request"]["source"]; isAdmin: boolean; isThread: boolean; - channelPrivacy?: 'private' | 'public'; + channelPrivacy?: "private" | "public"; policy: FixedTraceHybridPolicy; /** Internal test seam; production uses the unchanged quick-match function. */ quickMatcher?: HybridQuickMatcher; @@ -279,83 +386,143 @@ export function decideFixedTraceHybridRoute(input: { // Request facts are snapshotted before policy validation or matcher code. // A hostile request accessor therefore cannot run after a dispatch boundary. const policy = snapshotHybridPolicy(input); - if (snapshot.isAdmin) return { mode: 'llm_router_fallback', reason: 'admin_requires_router', plan: null }; - if (snapshot.isThread) return { mode: 'llm_router_fallback', reason: 'thread_context_requires_router', plan: null }; - if (snapshot.source === 'channel' && snapshot.channelPrivacy !== 'private') { - return { mode: 'llm_router_fallback', reason: 'channel_privacy_not_captured', plan: null }; + if (snapshot.isAdmin) + return { + mode: "llm_router_fallback", + reason: "admin_requires_router", + plan: null, + }; + if (snapshot.isThread) + return { + mode: "llm_router_fallback", + reason: "thread_context_requires_router", + plan: null, + }; + if (snapshot.source === "channel" && snapshot.channelPrivacy !== "private") { + return { + mode: "llm_router_fallback", + reason: "channel_privacy_not_captured", + plan: null, + }; } const safeForm = safeTerminalForm(snapshot); - if (!safeForm) return { mode: 'llm_router_fallback', reason: 'unsafe_or_ambiguous_message', plan: null }; + if (!safeForm) + return { + mode: "llm_router_fallback", + reason: "unsafe_or_ambiguous_message", + plan: null, + }; let matcher: HybridQuickMatcher; try { - const suppliedMatcher = Object.getOwnPropertyDescriptor(input, 'quickMatcher'); - if (suppliedMatcher && !('value' in suppliedMatcher)) throw new Error('quickMatcher accessor'); - if (suppliedMatcher?.value !== undefined && typeof suppliedMatcher.value !== 'function') { - throw new Error('quickMatcher is not a function'); + const suppliedMatcher = Object.getOwnPropertyDescriptor( + input, + "quickMatcher", + ); + if (suppliedMatcher && !("value" in suppliedMatcher)) + throw new Error("quickMatcher accessor"); + if ( + suppliedMatcher?.value !== undefined && + typeof suppliedMatcher.value !== "function" + ) { + throw new Error("quickMatcher is not a function"); } matcher = suppliedMatcher?.value ?? quickMatchRoutingContext; } catch { - return { mode: 'llm_router_fallback', reason: 'quick_match_exception', plan: null }; + return { + mode: "llm_router_fallback", + reason: "quick_match_exception", + plan: null, + }; } - let matchedAction: 'ignore' | 'react' | 'respond' | null; + let matchedAction: "ignore" | "react" | "respond" | null; let matchedEmoji: string | undefined; try { - const plan = matcher(Object.freeze({ - message: snapshot.message, - source: snapshot.source, - isThread: snapshot.isThread, - isAAOAdmin: snapshot.isAdmin, - })); + const plan = matcher( + Object.freeze({ + message: snapshot.message, + source: snapshot.source, + isThread: snapshot.isThread, + isAAOAdmin: snapshot.isAdmin, + }), + ); matchedAction = plan?.action ?? null; - matchedEmoji = plan?.action === 'react' ? plan.emoji : undefined; + matchedEmoji = plan?.action === "react" ? plan.emoji : undefined; } catch { - return { mode: 'llm_router_fallback', reason: 'quick_match_exception', plan: null }; + return { + mode: "llm_router_fallback", + reason: "quick_match_exception", + plan: null, + }; } - if (!matchedAction) return { mode: 'llm_router_fallback', reason: 'no_production_quick_match', plan: null }; - if (matchedAction === 'respond') { - return { mode: 'llm_router_fallback', reason: 'tool_or_mutation_capability_requires_router', plan: null }; + if (!matchedAction) + return { + mode: "llm_router_fallback", + reason: "no_production_quick_match", + plan: null, + }; + if (matchedAction === "respond") { + return { + mode: "llm_router_fallback", + reason: "tool_or_mutation_capability_requires_router", + plan: null, + }; } - if (matchedAction !== safeForm.action || !policy.localTerminalActions.includes(matchedAction)) { - return { mode: 'llm_router_fallback', reason: 'policy_disallows_terminal_action', plan: null }; + if ( + matchedAction !== safeForm.action || + !policy.localTerminalActions.includes(matchedAction) + ) { + return { + mode: "llm_router_fallback", + reason: "policy_disallows_terminal_action", + plan: null, + }; } - if (matchedAction === 'react' && matchedEmoji !== safeForm.emoji) { - return { mode: 'llm_router_fallback', reason: 'policy_disallows_terminal_action', plan: null }; + if (matchedAction === "react" && matchedEmoji !== safeForm.emoji) { + return { + mode: "llm_router_fallback", + reason: "policy_disallows_terminal_action", + plan: null, + }; } return { - mode: 'local_terminal', - reason: 'production_quick_match_terminal', - plan: safeForm.action === 'react' - ? Object.freeze({ - action: 'react' as const, - emoji: safeForm.emoji!, - reason: 'Reviewed exact harmless terminal form', - decision_method: 'quick_match' as const, - }) - : Object.freeze({ - action: 'ignore' as const, - reason: 'Reviewed exact harmless terminal form', - decision_method: 'quick_match' as const, - }), + mode: "local_terminal", + reason: "production_quick_match_terminal", + plan: + safeForm.action === "react" + ? Object.freeze({ + action: "react" as const, + emoji: safeForm.emoji!, + reason: "Reviewed exact harmless terminal form", + decision_method: "quick_match" as const, + }) + : Object.freeze({ + action: "ignore" as const, + reason: "Reviewed exact harmless terminal form", + decision_method: "quick_match" as const, + }), }; } export function fixedTraceArchitectureArm( - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceArchitectureArmProvenance { return FIXED_TRACE_ARCHITECTURE_ARMS[arm]; } export type FixedTraceToolDefinitionProvenance = - | 'fixture_local' - | 'evaluator_owned_production_definitions_simulated_receipts'; + "fixture_local" | "evaluator_owned_production_definitions_simulated_receipts"; /** Records what selected the candidate's visible tools for diagnostic replay. */ export interface FixedTraceToolUniverseProvenance { source: - | 'fixture_local_routed_replay' - | 'evaluator_owned_production_definitions_simulated_receipts' - | 'fixture_oracle'; - intentNarrowing: 'llm_router' | 'production_quick_match_or_llm_router' | 'not_applied' | 'fixture_oracle'; + | "fixture_local_routed_replay" + | "evaluator_owned_production_definitions_simulated_receipts" + | "fixture_oracle"; + intentNarrowing: + | "llm_router" + | "production_quick_match_or_llm_router" + | "not_applied" + | "fixture_oracle"; bounded: boolean; deployable: boolean; toolNames: readonly string[] | null; @@ -365,62 +532,79 @@ export interface FixedTraceToolUniverseProvenance { } export interface FixedTraceRequestThreadFactsProvenance { - source: 'not_applicable' | 'fixture_case_request_not_authenticated'; + source: "not_applicable" | "fixture_case_request_not_authenticated"; traceFacts: readonly Readonly<{ traceId: string; requestThreadFactsSha256: string; - provenance: 'fixture_case_request_not_authenticated'; + provenance: "fixture_case_request_not_authenticated"; }>[]; } export interface FixedTraceDirectRequestThreadFacts { - source: FixedTraceCase['request']['source']; + source: FixedTraceCase["request"]["source"]; isAAOAdmin: boolean; isThread: boolean; - channelPrivacy: 'private' | 'unknown'; - authentication: 'not_authenticated_fixture_claim'; - provenance: 'fixture_case_request_not_authenticated'; + channelPrivacy: "private" | "unknown"; + authentication: "not_authenticated_fixture_claim"; + provenance: "fixture_case_request_not_authenticated"; } function canonicalJson(value: unknown): string { - if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); - if (typeof value === 'number') return JSON.stringify(value); - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; - if (typeof value === 'object') { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (typeof value === "number") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { const record = value as Record; - return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; } - throw new Error('Cannot canonicalize a non-JSON request/thread fact'); + throw new Error("Cannot canonicalize a non-JSON request/thread fact"); } function sha256(value: unknown): string { - return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); + return createHash("sha256") + .update(canonicalJson(value), "utf8") + .digest("hex"); } /** Preserve fixture-visible facts exactly; never manufacture production auth/context. */ -export function fixedTraceDirectRequestThreadFacts(trace: FixedTraceCase): FixedTraceDirectRequestThreadFacts { +export function fixedTraceDirectRequestThreadFacts( + trace: FixedTraceCase, +): FixedTraceDirectRequestThreadFacts { return Object.freeze({ source: trace.request.source, isAAOAdmin: trace.request.isAdmin, isThread: (trace.request.threadContext?.length ?? 0) > 0, - channelPrivacy: trace.request.source === 'dm' ? 'private' : 'unknown', - authentication: 'not_authenticated_fixture_claim', - provenance: 'fixture_case_request_not_authenticated', + channelPrivacy: trace.request.source === "dm" ? "private" : "unknown", + authentication: "not_authenticated_fixture_claim", + provenance: "fixture_case_request_not_authenticated", }); } export function fixedTraceRequestThreadFactsProvenance( traceSuite: ReadonlyArray, - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceRequestThreadFactsProvenance { - if (arm !== 'direct_generation') return Object.freeze({ source: 'not_applicable', traceFacts: [] }); + if (arm !== "direct_generation") + return Object.freeze({ source: "not_applicable", traceFacts: [] }); return Object.freeze({ - source: 'fixture_case_request_not_authenticated', - traceFacts: Object.freeze(traceSuite.map((trace) => Object.freeze({ - traceId: trace.id, - requestThreadFactsSha256: sha256(fixedTraceDirectRequestThreadFacts(trace)), - provenance: 'fixture_case_request_not_authenticated' as const, - })).sort((left, right) => left.traceId.localeCompare(right.traceId))), + source: "fixture_case_request_not_authenticated", + traceFacts: Object.freeze( + traceSuite + .map((trace) => + Object.freeze({ + traceId: trace.id, + requestThreadFactsSha256: sha256( + fixedTraceDirectRequestThreadFacts(trace), + ), + provenance: "fixture_case_request_not_authenticated" as const, + }), + ) + .sort((left, right) => left.traceId.localeCompare(right.traceId)), + ), }); } @@ -430,60 +614,67 @@ export function fixedTraceRequestThreadFactsProvenance( * can reuse the production-equivalent executor. */ export interface FixedTraceExecutionEnvelopeProvenance { - source: 'fixture_expectation' | 'request_thread_facts_not_captured' | 'evaluator_owned_shared_request_thread_envelope' | 'fixture_oracle'; + source: + | "fixture_expectation" + | "request_thread_facts_not_captured" + | "evaluator_owned_shared_request_thread_envelope" + | "fixture_oracle"; deployable: boolean; } export function fixedTraceExecutionEnvelopeProvenance( - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceExecutionEnvelopeProvenance { - if (arm === 'direct_generation') return Object.freeze({ - source: 'evaluator_owned_shared_request_thread_envelope', - deployable: false, - }); - if (arm === 'oracle_route_diagnostic') return Object.freeze({ - source: 'fixture_oracle', - deployable: false, - }); - return Object.freeze({ source: 'fixture_expectation', deployable: false }); + if (arm === "direct_generation") + return Object.freeze({ + source: "evaluator_owned_shared_request_thread_envelope", + deployable: false, + }); + if (arm === "oracle_route_diagnostic") + return Object.freeze({ + source: "fixture_oracle", + deployable: false, + }); + return Object.freeze({ source: "fixture_expectation", deployable: false }); } export function fixedTraceToolUniverseProvenance( - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceToolUniverseProvenance { - if (arm === 'direct_generation') { + if (arm === "direct_generation") { return Object.freeze({ - source: 'evaluator_owned_production_definitions_simulated_receipts', - intentNarrowing: 'not_applied', + source: "evaluator_owned_production_definitions_simulated_receipts", + intentNarrowing: "not_applied", bounded: true, deployable: false, toolNames: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolNames, toolNamesSha256: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolNamesSha256, toolSchemaSha256: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolSchemaSha256, - definitionHandlerSha256: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.definitionHandlerSha256, + definitionHandlerSha256: + FIXED_TRACE_DIRECT_TOOL_UNIVERSE.definitionHandlerSha256, }); } - if (arm === 'oracle_route_diagnostic') { + if (arm === "oracle_route_diagnostic") { return Object.freeze({ - source: 'fixture_oracle', - intentNarrowing: 'fixture_oracle', + source: "fixture_oracle", + intentNarrowing: "fixture_oracle", bounded: true, deployable: false, toolNames: null, }); } - if (arm === 'deterministic_policy_llm_fallback_hybrid') { + if (arm === "deterministic_policy_llm_fallback_hybrid") { return Object.freeze({ - source: 'fixture_local_routed_replay', - intentNarrowing: 'production_quick_match_or_llm_router', + source: "fixture_local_routed_replay", + intentNarrowing: "production_quick_match_or_llm_router", bounded: true, deployable: false, toolNames: null, }); } return Object.freeze({ - source: 'fixture_local_routed_replay', - intentNarrowing: 'llm_router', + source: "fixture_local_routed_replay", + intentNarrowing: "llm_router", bounded: true, deployable: false, toolNames: null, @@ -491,19 +682,19 @@ export function fixedTraceToolUniverseProvenance( } export type FixedTraceDirectArmAdmissionReason = - | 'fixture_local_tool_definitions' - | 'request_thread_execution_envelope_not_captured' - | 'production_binding_contract_not_captured' - | 'request_thread_facts_not_authenticated' - | 'evaluator_simulated_receipt_handlers'; + | "fixture_local_tool_definitions" + | "request_thread_execution_envelope_not_captured" + | "production_binding_contract_not_captured" + | "request_thread_facts_not_authenticated" + | "evaluator_simulated_receipt_handlers"; export interface FixedTraceDirectToolUniverse extends FixedTraceToolUniverseProvenance { - surface: FixedTraceCase['request']['source']; + surface: FixedTraceCase["request"]["source"]; isAdmin: boolean; isThread: boolean; - channelPrivacy: 'private' | 'unknown'; + channelPrivacy: "private" | "unknown"; requestThreadFactsSha256: string; - requestThreadFactsProvenance: 'fixture_case_request_not_authenticated'; + requestThreadFactsProvenance: "fixture_case_request_not_authenticated"; } export interface FixedTraceDirectArmAdmission { @@ -522,7 +713,10 @@ function freezeAdmission( reasons: Object.freeze([...reasons]), universe: Object.freeze({ ...universe, - toolNames: universe.toolNames === null ? null : Object.freeze([...universe.toolNames]), + toolNames: + universe.toolNames === null + ? null + : Object.freeze([...universe.toolNames]), }), }); } @@ -533,10 +727,12 @@ function freezeAdmission( * grades. The evaluator cannot yet capture the authenticated definition / * handler intersection, and must not substitute a fixture-local subset. */ -export function deriveFixedTraceDirectToolUniverse(trace: FixedTraceCase): FixedTraceDirectToolUniverse { +export function deriveFixedTraceDirectToolUniverse( + trace: FixedTraceCase, +): FixedTraceDirectToolUniverse { const facts = fixedTraceDirectRequestThreadFacts(trace); return Object.freeze({ - ...fixedTraceToolUniverseProvenance('direct_generation'), + ...fixedTraceToolUniverseProvenance("direct_generation"), // These remain fixture claims, not production authentication. They are // retained for audit and bound to the cohort, never replaced by a DM. surface: facts.source, @@ -567,9 +763,13 @@ export function admitFixedTraceDirectArm( // contract or request/thread envelope, so it must reject before dispatch. void definitions; void definitionProvenance; - return freezeAdmission(false, [ - 'production_binding_contract_not_captured', - 'request_thread_facts_not_authenticated', - 'evaluator_simulated_receipt_handlers', - ], universe); + return freezeAdmission( + false, + [ + "production_binding_contract_not_captured", + "request_thread_facts_not_authenticated", + "evaluator_simulated_receipt_handlers", + ], + universe, + ); } diff --git a/server/src/addie/eval/fixed-trace-budget.ts b/server/src/addie/eval/fixed-trace-budget.ts index 52a7a12df6..e01102b465 100644 --- a/server/src/addie/eval/fixed-trace-budget.ts +++ b/server/src/addie/eval/fixed-trace-budget.ts @@ -6,13 +6,16 @@ import type { ModelUsage, NormalizedModelEvent, PreparedModelInvocation, -} from '../model-providers/model-provider.js'; +} from "../model-providers/model-provider.js"; import { GOOGLE_ROUTER_MODEL, isGoogleRouterModelRevision, -} from '../model-providers/google-generate-content-provider.js'; -import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION } from '../model-cost-pricing.js'; -import type { FixedTraceModelResolutionPolicy } from './fixed-trace-suite.js'; +} from "../model-providers/google-generate-content-provider.js"; +import { + GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_LUNA_PRICING, +} from "../model-cost-pricing.js"; +import type { FixedTraceModelResolutionPolicy } from "./fixed-trace-suite.js"; export interface FixedTraceBudgetPricing { inputUsdPerMillionTokens: number; @@ -22,29 +25,28 @@ export interface FixedTraceBudgetPricing { /** Null means this provider does not expose a separately billable cache write rate. */ cacheWriteUsdPerMillionTokens?: number | null; /** Cache reads and writes have independently recorded provider semantics. */ - cacheReadAccounting?: 'additive' | 'subset' | 'unsupported'; - cacheWriteAccounting?: 'additive' | 'subset' | 'unsupported'; + cacheReadAccounting?: "additive" | "subset" | "unsupported"; + cacheWriteAccounting?: "additive" | "subset" | "unsupported"; source: string; } export type FixedTraceBudgetRejectionReason = - | 'budget_exposure_unknown' - | 'soft_limit_exceeded'; + "budget_exposure_unknown" | "soft_limit_exceeded"; export class FixedTraceBudgetAdmissionError extends Error { - readonly terminalStatus = 'not_dispatched_budget' as const; + readonly terminalStatus = "not_dispatched_budget" as const; constructor( readonly reason: FixedTraceBudgetRejectionReason, readonly prepared: PreparedModelInvocation, ) { super(reason); - this.name = 'FixedTraceBudgetAdmissionError'; + this.name = "FixedTraceBudgetAdmissionError"; } } export interface FixedTraceBudgetSnapshot { - policy: 'soft_admission_target'; + policy: "soft_admission_target"; softMaxUsd: number; accountedSpendUsd: number; reservedUsd: number; @@ -63,58 +65,74 @@ export interface FixedTraceBudgetSnapshot { */ interface FixedTraceApprovedPricing extends FixedTraceBudgetPricing { readonly profileId: string; - readonly expectedProvider: ModelProvider['id']; + readonly expectedProvider: ModelProvider["id"]; readonly expectedModel: string; readonly modelResolutionPolicy: FixedTraceModelResolutionPolicy; } export function fixedTraceModelResolutionPolicy( - provider: ModelProvider['id'], + provider: ModelProvider["id"], model: string, ): FixedTraceModelResolutionPolicy { - return provider === 'google' && model === GOOGLE_ROUTER_MODEL - ? 'google_router_dated_revision_v1' - : 'exact_model_identity_v1'; + return provider === "google" && model === GOOGLE_ROUTER_MODEL + ? "google_router_dated_revision_v1" + : "exact_model_identity_v1"; } -const FIXED_TRACE_APPROVED_PRICING = Object.freeze(([ - { - expectedProvider: 'anthropic', expectedModel: 'claude-haiku-4-5', - profileId: 'anthropic-standard-2026-08:claude-haiku-4-5', - inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, - cacheReadUsdPerMillionTokens: 0.1, cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.', - modelResolutionPolicy: 'exact_model_identity_v1', - }, - { - expectedProvider: 'anthropic', expectedModel: 'claude-sonnet-5', - profileId: 'anthropic-standard-2026-08:claude-sonnet-5', - inputUsdPerMillionTokens: 3, outputUsdPerMillionTokens: 15, - cacheReadUsdPerMillionTokens: 0.3, cacheWriteUsdPerMillionTokens: 3.75, - cacheReadAccounting: 'additive', cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Sonnet 5 standard, refreshed August 2026.', - modelResolutionPolicy: 'exact_model_identity_v1', - }, - { - expectedProvider: 'openai', expectedModel: 'gpt-5.6-luna', - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', - inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, - cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset', cacheWriteAccounting: 'unsupported', - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', - modelResolutionPolicy: 'exact_model_identity_v1', - }, - { - expectedProvider: 'google', expectedModel: GOOGLE_ROUTER_MODEL, - profileId: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - inputUsdPerMillionTokens: 0.75, outputUsdPerMillionTokens: 3.75, - cacheReadUsdPerMillionTokens: 0.075, cacheWriteUsdPerMillionTokens: 0.75, - cacheReadAccounting: 'subset', cacheWriteAccounting: 'additive', - source: 'Google Gemini 3.7 Flash introductory standard, checked 2026-08-25.', - modelResolutionPolicy: 'google_router_dated_revision_v1', - }, -] satisfies readonly FixedTraceApprovedPricing[]).map((entry) => Object.freeze(entry))); +const FIXED_TRACE_APPROVED_PRICING = Object.freeze( + ( + [ + { + expectedProvider: "anthropic", + expectedModel: "claude-haiku-4-5", + profileId: "anthropic-standard-2026-08:claude-haiku-4-5", + inputUsdPerMillionTokens: 1, + outputUsdPerMillionTokens: 5, + cacheReadUsdPerMillionTokens: 0.1, + cacheWriteUsdPerMillionTokens: 1.25, + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + source: + "Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.", + modelResolutionPolicy: "exact_model_identity_v1", + }, + { + expectedProvider: "anthropic", + expectedModel: "claude-sonnet-5", + profileId: "anthropic-standard-2026-08:claude-sonnet-5", + inputUsdPerMillionTokens: 3, + outputUsdPerMillionTokens: 15, + cacheReadUsdPerMillionTokens: 0.3, + cacheWriteUsdPerMillionTokens: 3.75, + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + source: + "Repository Anthropic pricing table: Claude Sonnet 5 standard, refreshed August 2026.", + modelResolutionPolicy: "exact_model_identity_v1", + }, + { + expectedProvider: "openai", + expectedModel: "gpt-5.6-luna", + ...OPENAI_GPT_5_6_LUNA_PRICING, + modelResolutionPolicy: "exact_model_identity_v1", + }, + { + expectedProvider: "google", + expectedModel: GOOGLE_ROUTER_MODEL, + profileId: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + inputUsdPerMillionTokens: 0.75, + outputUsdPerMillionTokens: 3.75, + cacheReadUsdPerMillionTokens: 0.075, + cacheWriteUsdPerMillionTokens: 0.75, + cacheReadAccounting: "subset", + cacheWriteAccounting: "additive", + source: + "Google Gemini 3.7 Flash introductory standard, checked 2026-08-25.", + modelResolutionPolicy: "google_router_dated_revision_v1", + }, + ] satisfies readonly FixedTraceApprovedPricing[] + ).map((entry) => Object.freeze(entry)), +); /** * The complete live approval surface. It is intentionally inspectable for @@ -122,22 +140,24 @@ const FIXED_TRACE_APPROVED_PRICING = Object.freeze(([ * these descriptive values. */ export function fixedTraceApprovedPricingProfiles(): readonly Readonly<{ - expectedProvider: ModelProvider['id']; + expectedProvider: ModelProvider["id"]; expectedModel: string; profileId: string; source: string; }>[] { - return FIXED_TRACE_APPROVED_PRICING.map((entry) => Object.freeze({ - expectedProvider: entry.expectedProvider, - expectedModel: entry.expectedModel, - profileId: entry.profileId, - source: entry.source, - })); + return FIXED_TRACE_APPROVED_PRICING.map((entry) => + Object.freeze({ + expectedProvider: entry.expectedProvider, + expectedModel: entry.expectedModel, + profileId: entry.profileId, + source: entry.source, + }), + ); } /** Opaque, module-branded policy produced only from the approved registry. */ export interface FixedTraceResponsePricingPolicy { - readonly expectedProvider: ModelProvider['id']; + readonly expectedProvider: ModelProvider["id"]; readonly expectedModel: string; readonly pricingProfileId: string; readonly modelResolutionPolicy: FixedTraceModelResolutionPolicy; @@ -152,36 +172,46 @@ function sameApprovedPricing( entry: FixedTraceApprovedPricing, pricing: FixedTraceBudgetPricing & { readonly profileId: string }, ): boolean { - return entry.profileId === pricing.profileId - && entry.inputUsdPerMillionTokens === pricing.inputUsdPerMillionTokens - && entry.outputUsdPerMillionTokens === pricing.outputUsdPerMillionTokens - && entry.cacheReadUsdPerMillionTokens === pricing.cacheReadUsdPerMillionTokens - && entry.cacheWriteUsdPerMillionTokens === pricing.cacheWriteUsdPerMillionTokens - && entry.cacheReadAccounting === pricing.cacheReadAccounting - && entry.cacheWriteAccounting === pricing.cacheWriteAccounting - && entry.source === pricing.source; + return ( + entry.profileId === pricing.profileId && + entry.inputUsdPerMillionTokens === pricing.inputUsdPerMillionTokens && + entry.outputUsdPerMillionTokens === pricing.outputUsdPerMillionTokens && + entry.cacheReadUsdPerMillionTokens === + pricing.cacheReadUsdPerMillionTokens && + entry.cacheWriteUsdPerMillionTokens === + pricing.cacheWriteUsdPerMillionTokens && + entry.cacheReadAccounting === pricing.cacheReadAccounting && + entry.cacheWriteAccounting === pricing.cacheWriteAccounting && + entry.source === pricing.source + ); } function approvedResponsePricing( policy: FixedTraceResponsePricingPolicy, ): FixedTraceApprovedPricing { const approved = approvedResponsePricingPolicies.get(policy); - if (!approved) throw new Error('Fixed trace returned-model pricing policy is not evaluator approved'); + if (!approved) + throw new Error( + "Fixed trace returned-model pricing policy is not evaluator approved", + ); return approved; } export function fixedTraceResponsePricingPolicy( - expectedProvider: ModelProvider['id'], + expectedProvider: ModelProvider["id"], expectedModel: string, pricing: FixedTraceBudgetPricing & { readonly profileId: string }, ): FixedTraceResponsePricingPolicy { - const approved = FIXED_TRACE_APPROVED_PRICING.find((entry) => ( - entry.expectedProvider === expectedProvider - && entry.expectedModel === expectedModel - && entry.modelResolutionPolicy === fixedTraceModelResolutionPolicy(expectedProvider, expectedModel) - && sameApprovedPricing(entry, pricing) - )); - if (!approved) throw new Error('Fixed trace pricing profile is not evaluator approved'); + const approved = FIXED_TRACE_APPROVED_PRICING.find( + (entry) => + entry.expectedProvider === expectedProvider && + entry.expectedModel === expectedModel && + entry.modelResolutionPolicy === + fixedTraceModelResolutionPolicy(expectedProvider, expectedModel) && + sameApprovedPricing(entry, pricing), + ); + if (!approved) + throw new Error("Fixed trace pricing profile is not evaluator approved"); const policy = Object.freeze({ expectedProvider: approved.expectedProvider, expectedModel: approved.expectedModel, @@ -199,9 +229,11 @@ export function fixedTraceResponseUsesPricingPolicy( const approved = approvedResponsePricing(policy); if (response.provider !== policy.expectedProvider) return false; if (response.model === policy.expectedModel) return true; - return policy.modelResolutionPolicy === 'google_router_dated_revision_v1' - && approved.profileId === GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION - && isGoogleRouterModelRevision(response.model); + return ( + policy.modelResolutionPolicy === "google_router_dated_revision_v1" && + approved.profileId === GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION && + isGoogleRouterModelRevision(response.model) + ); } interface Reservation { @@ -219,11 +251,11 @@ interface BudgetedProviderBinding { interface BudgetedDelegateIdentity { readonly delegate: ModelProvider; - readonly id: ModelProvider['id']; - readonly capabilities: ModelProvider['capabilities']; - readonly prepare: ModelProvider['prepare']; - readonly respond: ModelProvider['respond']; - readonly deriveProviderToolReceipt?: ModelProvider['deriveProviderToolReceipt']; + readonly id: ModelProvider["id"]; + readonly capabilities: ModelProvider["capabilities"]; + readonly prepare: ModelProvider["prepare"]; + readonly respond: ModelProvider["respond"]; + readonly deriveProviderToolReceipt?: ModelProvider["deriveProviderToolReceipt"]; } // This is deliberately not an instance field or a public predicate. The @@ -232,10 +264,14 @@ interface BudgetedDelegateIdentity { // replacing the dispatch path. const budgetedProviderBindings = new WeakMap(); const exclusiveBudgetLeases = new WeakMap(); -const exclusiveCloneIdentities = new WeakMap(); +const exclusiveCloneIdentities = new WeakMap< + object, + BudgetedDelegateIdentity +>(); function deepFreeze(value: T): T { - if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; + if (typeof value !== "object" || value === null || Object.isFrozen(value)) + return value; for (const nested of Object.values(value)) deepFreeze(nested); return Object.freeze(value); } @@ -244,16 +280,25 @@ function snapshotPricing(pricing: T): T { return Object.freeze({ ...pricing }) as T; } -function snapshotDelegateIdentity(delegate: ModelProvider): BudgetedDelegateIdentity { +function snapshotDelegateIdentity( + delegate: ModelProvider, +): BudgetedDelegateIdentity { // Read all mutable delegate surface once. Lease cloning reuses this sealed // identity rather than re-reading a delegate getter after preflight. const id = delegate.id; - const capabilities = deepFreeze(structuredClone(delegate.capabilities)) as ModelProvider['capabilities']; + const capabilities = deepFreeze( + structuredClone(delegate.capabilities), + ) as ModelProvider["capabilities"]; const prepare = delegate.prepare; const respond = delegate.respond; const deriveProviderToolReceipt = delegate.deriveProviderToolReceipt; - if (typeof id !== 'string' || !id.trim() || typeof prepare !== 'function' || typeof respond !== 'function') { - throw new Error('Fixed trace budget delegate identity is invalid'); + if ( + typeof id !== "string" || + !id.trim() || + typeof prepare !== "function" || + typeof respond !== "function" + ) { + throw new Error("Fixed trace budget delegate identity is invalid"); } return Object.freeze({ delegate, @@ -265,43 +310,61 @@ function snapshotDelegateIdentity(delegate: ModelProvider): BudgetedDelegateIden }); } -function samePricing(left: FixedTraceBudgetPricing, right: FixedTraceBudgetPricing): boolean { - return left.inputUsdPerMillionTokens === right.inputUsdPerMillionTokens - && left.outputUsdPerMillionTokens === right.outputUsdPerMillionTokens - && left.cacheReadUsdPerMillionTokens === right.cacheReadUsdPerMillionTokens - && left.cacheWriteUsdPerMillionTokens === right.cacheWriteUsdPerMillionTokens - && left.cacheReadAccounting === right.cacheReadAccounting - && left.cacheWriteAccounting === right.cacheWriteAccounting - && left.source === right.source; +function samePricing( + left: FixedTraceBudgetPricing, + right: FixedTraceBudgetPricing, +): boolean { + return ( + left.inputUsdPerMillionTokens === right.inputUsdPerMillionTokens && + left.outputUsdPerMillionTokens === right.outputUsdPerMillionTokens && + left.cacheReadUsdPerMillionTokens === right.cacheReadUsdPerMillionTokens && + left.cacheWriteUsdPerMillionTokens === + right.cacheWriteUsdPerMillionTokens && + left.cacheReadAccounting === right.cacheReadAccounting && + left.cacheWriteAccounting === right.cacheWriteAccounting && + left.source === right.source + ); } function sameResponsePricingPolicy( left: FixedTraceResponsePricingPolicy, right: FixedTraceResponsePricingPolicy, ): boolean { - return left.expectedProvider === right.expectedProvider - && left.expectedModel === right.expectedModel - && left.pricingProfileId === right.pricingProfileId - && left.modelResolutionPolicy === right.modelResolutionPolicy; + return ( + left.expectedProvider === right.expectedProvider && + left.expectedModel === right.expectedModel && + left.pricingProfileId === right.pricingProfileId && + left.modelResolutionPolicy === right.modelResolutionPolicy + ); } -export function validateFixedTracePricing(pricing: FixedTraceBudgetPricing): void { +export function validateFixedTracePricing( + pricing: FixedTraceBudgetPricing, +): void { if ( - !Number.isFinite(pricing.inputUsdPerMillionTokens) - || pricing.inputUsdPerMillionTokens < 0 - || !Number.isFinite(pricing.outputUsdPerMillionTokens) - || pricing.outputUsdPerMillionTokens < 0 - || !pricing.source.trim() - ) throw new Error('Fixed trace budget pricing is invalid'); - for (const rate of [pricing.cacheReadUsdPerMillionTokens, pricing.cacheWriteUsdPerMillionTokens]) { - if (rate !== undefined && rate !== null && (!Number.isFinite(rate) || rate < 0)) { - throw new Error('Fixed trace cache pricing is invalid'); + !Number.isFinite(pricing.inputUsdPerMillionTokens) || + pricing.inputUsdPerMillionTokens < 0 || + !Number.isFinite(pricing.outputUsdPerMillionTokens) || + pricing.outputUsdPerMillionTokens < 0 || + !pricing.source.trim() + ) + throw new Error("Fixed trace budget pricing is invalid"); + for (const rate of [ + pricing.cacheReadUsdPerMillionTokens, + pricing.cacheWriteUsdPerMillionTokens, + ]) { + if ( + rate !== undefined && + rate !== null && + (!Number.isFinite(rate) || rate < 0) + ) { + throw new Error("Fixed trace cache pricing is invalid"); } } } function requestBytes(prepared: PreparedModelInvocation): number { - return Buffer.byteLength(JSON.stringify(prepared.providerRequest), 'utf8'); + return Buffer.byteLength(JSON.stringify(prepared.providerRequest), "utf8"); } /** @@ -316,41 +379,54 @@ export function fixedTraceEstimatedCostUsd( validateFixedTracePricing(pricing); const { inputTokens, outputTokens } = usage; if ( - !Number.isSafeInteger(inputTokens) - || inputTokens < 0 - || !Number.isSafeInteger(outputTokens) - || outputTokens < 0 - ) throw new Error('Fixed trace budget usage is invalid'); + !Number.isSafeInteger(inputTokens) || + inputTokens < 0 || + !Number.isSafeInteger(outputTokens) || + outputTokens < 0 + ) + throw new Error("Fixed trace budget usage is invalid"); const cacheReadTokens = usage.cacheReadTokens ?? 0; const cacheWriteTokens = usage.cacheWriteTokens ?? 0; if ( - !Number.isSafeInteger(cacheReadTokens) || cacheReadTokens < 0 - || !Number.isSafeInteger(cacheWriteTokens) || cacheWriteTokens < 0 - ) throw new Error('Fixed trace cache usage is invalid'); - const readAccounting = pricing.cacheReadAccounting ?? 'unsupported'; - const writeAccounting = pricing.cacheWriteAccounting ?? 'unsupported'; - if (cacheReadTokens > 0 && readAccounting === 'unsupported') throw new Error('Fixed trace cache read accounting is unavailable'); - if (cacheWriteTokens > 0 && writeAccounting === 'unsupported') throw new Error('Fixed trace cache write accounting is unavailable'); - if (readAccounting === 'subset' && cacheReadTokens > inputTokens) throw new Error('Fixed trace subset cache read usage is invalid'); + !Number.isSafeInteger(cacheReadTokens) || + cacheReadTokens < 0 || + !Number.isSafeInteger(cacheWriteTokens) || + cacheWriteTokens < 0 + ) + throw new Error("Fixed trace cache usage is invalid"); + const readAccounting = pricing.cacheReadAccounting ?? "unsupported"; + const writeAccounting = pricing.cacheWriteAccounting ?? "unsupported"; + if (cacheReadTokens > 0 && readAccounting === "unsupported") + throw new Error("Fixed trace cache read accounting is unavailable"); + if (cacheWriteTokens > 0 && writeAccounting === "unsupported") + throw new Error("Fixed trace cache write accounting is unavailable"); + if (readAccounting === "subset" && cacheReadTokens > inputTokens) + throw new Error("Fixed trace subset cache read usage is invalid"); // A subset read and additive write (Google's profile) is valid. Two subset // buckets must jointly fit the provider's normalized input total. - if (readAccounting === 'subset' && writeAccounting === 'subset' && cacheReadTokens + cacheWriteTokens > inputTokens) { - throw new Error('Fixed trace subset cache usage is invalid'); + if ( + readAccounting === "subset" && + writeAccounting === "subset" && + cacheReadTokens + cacheWriteTokens > inputTokens + ) { + throw new Error("Fixed trace subset cache usage is invalid"); } if (cacheReadTokens > 0 && pricing.cacheReadUsdPerMillionTokens == null) { - throw new Error('Fixed trace cache read pricing is unavailable'); + throw new Error("Fixed trace cache read pricing is unavailable"); } if (cacheWriteTokens > 0 && pricing.cacheWriteUsdPerMillionTokens == null) { - throw new Error('Fixed trace cache write pricing is unavailable'); + throw new Error("Fixed trace cache write pricing is unavailable"); } return ( - (inputTokens - - (readAccounting === 'subset' ? cacheReadTokens : 0) - - (writeAccounting === 'subset' ? cacheWriteTokens : 0)) * pricing.inputUsdPerMillionTokens - + outputTokens * pricing.outputUsdPerMillionTokens - + cacheReadTokens * (pricing.cacheReadUsdPerMillionTokens ?? 0) - + cacheWriteTokens * (pricing.cacheWriteUsdPerMillionTokens ?? 0) - ) / 1_000_000; + ((inputTokens - + (readAccounting === "subset" ? cacheReadTokens : 0) - + (writeAccounting === "subset" ? cacheWriteTokens : 0)) * + pricing.inputUsdPerMillionTokens + + outputTokens * pricing.outputUsdPerMillionTokens + + cacheReadTokens * (pricing.cacheReadUsdPerMillionTokens ?? 0) + + cacheWriteTokens * (pricing.cacheWriteUsdPerMillionTokens ?? 0)) / + 1_000_000 + ); } /** @@ -371,7 +447,7 @@ export class FixedTraceBudget { constructor(readonly softMaxUsd: number) { if (!Number.isFinite(softMaxUsd) || softMaxUsd <= 0) { - throw new RangeError('Fixed trace soft budget must be positive'); + throw new RangeError("Fixed trace soft budget must be positive"); } } @@ -384,18 +460,25 @@ export class FixedTraceBudget { validateFixedTracePricing(pricing); const exclusiveLease = exclusiveBudgetLeases.get(this); if (exclusiveLease !== undefined && lease !== exclusiveLease) { - throw new Error('Fixed trace budget is reserved for an exclusive diagnostic run'); + throw new Error( + "Fixed trace budget is reserved for an exclusive diagnostic run", + ); } if (!Number.isSafeInteger(maxOutputTokens) || maxOutputTokens < 1) { - throw new RangeError('Fixed trace output reserve must be a positive integer'); + throw new RangeError( + "Fixed trace output reserve must be a positive integer", + ); } if (this.exposureUnknown) { this.budgetRejectedCalls++; - throw new FixedTraceBudgetAdmissionError('budget_exposure_unknown', prepared); + throw new FixedTraceBudgetAdmissionError( + "budget_exposure_unknown", + prepared, + ); } if (this.admissionClosed) { this.budgetRejectedCalls++; - throw new FixedTraceBudgetAdmissionError('soft_limit_exceeded', prepared); + throw new FixedTraceBudgetAdmissionError("soft_limit_exceeded", prepared); } // Request bytes are a deliberately high token bound for the request. An // additive cache bucket is separately billable, so reserve that same @@ -403,16 +486,21 @@ export class FixedTraceBudget { // by inputTokens. This keeps the pre-dispatch reserve conservative under // the recorded, fingerprinted cache formula. const inputTokens = requestBytes(prepared); - const usd = fixedTraceEstimatedCostUsd({ - inputTokens, - outputTokens: maxOutputTokens, - cacheReadTokens: pricing.cacheReadAccounting === 'additive' ? inputTokens : 0, - cacheWriteTokens: pricing.cacheWriteAccounting === 'additive' ? inputTokens : 0, - }, pricing); + const usd = fixedTraceEstimatedCostUsd( + { + inputTokens, + outputTokens: maxOutputTokens, + cacheReadTokens: + pricing.cacheReadAccounting === "additive" ? inputTokens : 0, + cacheWriteTokens: + pricing.cacheWriteAccounting === "additive" ? inputTokens : 0, + }, + pricing, + ); if (this.accountedSpendUsd + this.reservedUsd + usd > this.softMaxUsd) { this.admissionClosed = true; this.budgetRejectedCalls++; - throw new FixedTraceBudgetAdmissionError('soft_limit_exceeded', prepared); + throw new FixedTraceBudgetAdmissionError("soft_limit_exceeded", prepared); } this.reservedUsd += usd; return { usd, active: true }; @@ -445,13 +533,16 @@ export class FixedTraceBudget { snapshot(): FixedTraceBudgetSnapshot { return Object.freeze({ - policy: 'soft_admission_target', + policy: "soft_admission_target", softMaxUsd: this.softMaxUsd, accountedSpendUsd: this.accountedSpendUsd, reservedUsd: this.reservedUsd, remainingUsd: this.exposureUnknown ? null - : Math.max(0, this.softMaxUsd - this.accountedSpendUsd - this.reservedUsd), + : Math.max( + 0, + this.softMaxUsd - this.accountedSpendUsd - this.reservedUsd, + ), dispatchedCalls: this.dispatchedCalls, completedCalls: this.completedCalls, budgetRejectedCalls: this.budgetRejectedCalls, @@ -461,7 +552,8 @@ export class FixedTraceBudget { } private requireActive(reservation: Reservation): void { - if (!reservation.active) throw new Error('Fixed trace budget reservation is inactive'); + if (!reservation.active) + throw new Error("Fixed trace budget reservation is inactive"); } private release(reservation: Reservation): void { @@ -473,9 +565,9 @@ export class FixedTraceBudget { /** Model-provider decorator that applies a shared budget at the dispatch edge. */ export class BudgetedFixedTraceProvider implements ModelProvider { - readonly id: ModelProvider['id']; - readonly capabilities: ModelProvider['capabilities']; - readonly deriveProviderToolReceipt?: ModelProvider['deriveProviderToolReceipt']; + readonly id: ModelProvider["id"]; + readonly capabilities: ModelProvider["capabilities"]; + readonly deriveProviderToolReceipt?: ModelProvider["deriveProviderToolReceipt"]; readonly #delegate: BudgetedDelegateIdentity; readonly #budget: FixedTraceBudget; @@ -491,17 +583,23 @@ export class BudgetedFixedTraceProvider implements ModelProvider { ) { const approvedPricing = approvedResponsePricing(responsePricingPolicy); if (!sameApprovedPricing(approvedPricing, pricing)) { - throw new Error('Fixed trace budget pricing does not match its evaluator-approved policy'); + throw new Error( + "Fixed trace budget pricing does not match its evaluator-approved policy", + ); } - const clonedIdentity = cloneIdentityToken === undefined - ? undefined - : exclusiveCloneIdentities.get(cloneIdentityToken); + const clonedIdentity = + cloneIdentityToken === undefined + ? undefined + : exclusiveCloneIdentities.get(cloneIdentityToken); if (cloneIdentityToken !== undefined && !clonedIdentity) { - throw new Error('Fixed trace budget clone identity is unavailable'); + throw new Error("Fixed trace budget clone identity is unavailable"); } - const delegateIdentity = clonedIdentity ?? snapshotDelegateIdentity(delegate); + const delegateIdentity = + clonedIdentity ?? snapshotDelegateIdentity(delegate); if (delegateIdentity.id !== responsePricingPolicy.expectedProvider) { - throw new Error('Fixed trace budget delegate identity does not match its pricing policy'); + throw new Error( + "Fixed trace budget delegate identity does not match its pricing policy", + ); } this.#delegate = delegateIdentity; this.#budget = budget; @@ -510,7 +608,8 @@ export class BudgetedFixedTraceProvider implements ModelProvider { this.id = delegateIdentity.id; this.capabilities = delegateIdentity.capabilities; if (delegateIdentity.deriveProviderToolReceipt) { - this.deriveProviderToolReceipt = delegateIdentity.deriveProviderToolReceipt; + this.deriveProviderToolReceipt = + delegateIdentity.deriveProviderToolReceipt; } budgetedProviderBindings.set(this, { budget, @@ -545,7 +644,12 @@ export class BudgetedFixedTraceProvider implements ModelProvider { ...options, beforeDispatch: async (prepared) => { this.assertPreparedIdentity(prepared); - reservation = this.#budget.reserve(prepared, request.maxOutputTokens, this.#pricing, lease ?? undefined); + reservation = this.#budget.reserve( + prepared, + request.maxOutputTokens, + this.#pricing, + lease ?? undefined, + ); try { await options.beforeDispatch?.(prepared); } catch (error) { @@ -557,16 +661,23 @@ export class BudgetedFixedTraceProvider implements ModelProvider { dispatchStarted = true; }, })) { - if (event.type === 'response_complete') { + if (event.type === "response_complete") { if (!reservation || !dispatchStarted) { - throw new Error('Fixed trace provider completed without dispatch admission'); + throw new Error( + "Fixed trace provider completed without dispatch admission", + ); } // The delegate still owns `event.response` and may mutate it when // the iterator resumes after this yield. One evaluator-owned frozen // snapshot is therefore the sole terminal response used for // approval, settlement, and the outward event. const response = deepFreeze(structuredClone(event.response)); - if (fixedTraceResponseUsesPricingPolicy(this.#responsePricingPolicy, response)) { + if ( + fixedTraceResponseUsesPricingPolicy( + this.#responsePricingPolicy, + response, + ) + ) { this.#budget.complete(reservation, response.usage, this.#pricing); } else { // Do not settle an unapproved returned identity at the requested @@ -576,7 +687,7 @@ export class BudgetedFixedTraceProvider implements ModelProvider { this.#budget.markExposureUnknown(reservation); } settled = true; - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; continue; } yield event; @@ -591,15 +702,20 @@ export class BudgetedFixedTraceProvider implements ModelProvider { private assertRequestIdentity(request: ModelRequest): void { if (request.model !== this.#responsePricingPolicy.expectedModel) { - throw new Error('Fixed trace budget request model does not match its pricing policy'); + throw new Error( + "Fixed trace budget request model does not match its pricing policy", + ); } } private assertPreparedIdentity(prepared: PreparedModelInvocation): void { if ( - prepared.provider !== this.id - || prepared.model !== this.#responsePricingPolicy.expectedModel - ) throw new Error('Fixed trace budget prepared invocation identity does not match its pricing policy'); + prepared.provider !== this.id || + prepared.model !== this.#responsePricingPolicy.expectedModel + ) + throw new Error( + "Fixed trace budget prepared invocation identity does not match its pricing policy", + ); } static cloneForExclusiveDiagnosticRun( @@ -607,7 +723,8 @@ export class BudgetedFixedTraceProvider implements ModelProvider { lease: object, ): BudgetedFixedTraceProvider { const binding = budgetedProviderBindings.get(source); - if (!binding) throw new Error('Fixed trace budget wrapper binding is unavailable'); + if (!binding) + throw new Error("Fixed trace budget wrapper binding is unavailable"); const cloneIdentityToken = Object.freeze({}); exclusiveCloneIdentities.set(cloneIdentityToken, binding.delegate); let clone: BudgetedFixedTraceProvider; @@ -623,14 +740,17 @@ export class BudgetedFixedTraceProvider implements ModelProvider { exclusiveCloneIdentities.delete(cloneIdentityToken); } const cloneBinding = budgetedProviderBindings.get(clone); - if (!cloneBinding) throw new Error('Fixed trace budget wrapper binding is unavailable'); + if (!cloneBinding) + throw new Error("Fixed trace budget wrapper binding is unavailable"); cloneBinding.lease = lease; return clone; } } -const budgetedFixedTraceProviderPrepare = BudgetedFixedTraceProvider.prototype.prepare; -const budgetedFixedTraceProviderRespond = BudgetedFixedTraceProvider.prototype.respond; +const budgetedFixedTraceProviderPrepare = + BudgetedFixedTraceProvider.prototype.prepare; +const budgetedFixedTraceProviderRespond = + BudgetedFixedTraceProvider.prototype.respond; Object.freeze(BudgetedFixedTraceProvider.prototype); Object.freeze(BudgetedFixedTraceProvider); @@ -647,15 +767,26 @@ export function isTrustedBudgetedFixedTraceProvider( ): boolean { const binding = budgetedProviderBindings.get(provider); if ( - binding?.budget !== budget - || !samePricing(binding.pricing, pricing) - || !sameResponsePricingPolicy(binding.responsePricingPolicy, responsePricingPolicy) - ) return false; - if (Object.getPrototypeOf(provider) !== BudgetedFixedTraceProvider.prototype) return false; - if ((provider as unknown as { constructor: unknown }).constructor !== BudgetedFixedTraceProvider) return false; - return Object.isFrozen(provider) - && provider.prepare === budgetedFixedTraceProviderPrepare - && provider.respond === budgetedFixedTraceProviderRespond; + binding?.budget !== budget || + !samePricing(binding.pricing, pricing) || + !sameResponsePricingPolicy( + binding.responsePricingPolicy, + responsePricingPolicy, + ) + ) + return false; + if (Object.getPrototypeOf(provider) !== BudgetedFixedTraceProvider.prototype) + return false; + if ( + (provider as unknown as { constructor: unknown }).constructor !== + BudgetedFixedTraceProvider + ) + return false; + return ( + Object.isFrozen(provider) && + provider.prepare === budgetedFixedTraceProviderPrepare && + provider.respond === budgetedFixedTraceProviderRespond + ); } export interface FixedTraceBudgetDiagnosticLease { @@ -675,38 +806,52 @@ export function claimFixedTraceBudgetDiagnosticLease( ): FixedTraceBudgetDiagnosticLease { const snapshot = budget.snapshot(); if ( - snapshot.accountedSpendUsd !== 0 - || snapshot.reservedUsd !== 0 - || snapshot.dispatchedCalls !== 0 - || snapshot.completedCalls !== 0 - || snapshot.budgetRejectedCalls !== 0 - || snapshot.admissionClosed - || snapshot.exposureUnknown - || exclusiveBudgetLeases.has(budget) - ) throw new Error('Fixed trace diagnostic budget must be pristine and exclusively claimed'); + snapshot.accountedSpendUsd !== 0 || + snapshot.reservedUsd !== 0 || + snapshot.dispatchedCalls !== 0 || + snapshot.completedCalls !== 0 || + snapshot.budgetRejectedCalls !== 0 || + snapshot.admissionClosed || + snapshot.exposureUnknown || + exclusiveBudgetLeases.has(budget) + ) + throw new Error( + "Fixed trace diagnostic budget must be pristine and exclusively claimed", + ); const lease = Object.freeze({}); const clones = new Map(); for (const provider of providers) { if (clones.has(provider)) continue; const binding = budgetedProviderBindings.get(provider); - if (!binding || !isTrustedBudgetedFixedTraceProvider( - provider, - budget, - binding.pricing, - binding.responsePricingPolicy, - )) { - throw new Error('Fixed trace diagnostic provider is not an authenticated budget wrapper'); + if ( + !binding || + !isTrustedBudgetedFixedTraceProvider( + provider, + budget, + binding.pricing, + binding.responsePricingPolicy, + ) + ) { + throw new Error( + "Fixed trace diagnostic provider is not an authenticated budget wrapper", + ); } - clones.set(provider, BudgetedFixedTraceProvider.cloneForExclusiveDiagnosticRun( - provider as BudgetedFixedTraceProvider, - lease, - )); + clones.set( + provider, + BudgetedFixedTraceProvider.cloneForExclusiveDiagnosticRun( + provider as BudgetedFixedTraceProvider, + lease, + ), + ); } const diagnosticLease = Object.freeze({ providerFor(provider: ModelProvider): BudgetedFixedTraceProvider { const clone = clones.get(provider); - if (!clone) throw new Error('Fixed trace diagnostic provider is missing from its exclusive lease'); + if (!clone) + throw new Error( + "Fixed trace diagnostic provider is missing from its exclusive lease", + ); return clone; }, }); @@ -714,16 +859,19 @@ export function claimFixedTraceBudgetDiagnosticLease( const sourceBinding = budgetedProviderBindings.get(source); const cloneBinding = budgetedProviderBindings.get(clone); if ( - !sourceBinding - || !cloneBinding - || cloneBinding.delegate !== sourceBinding.delegate - || !isTrustedBudgetedFixedTraceProvider( + !sourceBinding || + !cloneBinding || + cloneBinding.delegate !== sourceBinding.delegate || + !isTrustedBudgetedFixedTraceProvider( clone, budget, sourceBinding.pricing, sourceBinding.responsePricingPolicy, ) - ) throw new Error('Fixed trace diagnostic clone identity is not authenticated'); + ) + throw new Error( + "Fixed trace diagnostic clone identity is not authenticated", + ); } // Diagnostic plans can additionally validate their cloned stages here. The // callback has no asynchronous boundary and runs before the lease becomes diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index 16fcf2eb52..7c4e26cfcc 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -1,945 +1,1102 @@ -import { createHash } from 'node:crypto'; -import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; +import { createHash } from "node:crypto"; +import { CLAUDE_PRICING_VERSION } from "../claude-pricing.js"; import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - OPENAI_GPT_5_6_LUNA_PRICING_VERSION, -} from '../model-cost-pricing.js'; -import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; -import { OPENAI_ROUTER_MODEL } from '../model-providers/openai-responses-provider.js'; + OPENAI_GPT_5_6_LUNA_PRICING, +} from "../model-cost-pricing.js"; +import { ANTHROPIC_PROVIDER_CAPABILITIES } from "../model-providers/anthropic-provider.js"; +import { + GOOGLE_GENERATE_CONTENT_CAPABILITIES, + GOOGLE_ROUTER_MODEL, +} from "../model-providers/google-generate-content-provider.js"; +import type { + ModelProviderId, + ModelReasoningEffort, +} from "../model-providers/model-provider.js"; +import { + OPENAI_RESPONSES_CAPABILITIES, + OPENAI_ROUTER_MODEL, +} from "../model-providers/openai-responses-provider.js"; +import { ANTHROPIC_ROUTER_CAPABILITIES } from "../model-providers/anthropic-router-provider.js"; +import { + decideFixedTraceHybridRoute, + fixedTraceHybridPolicy, + type FixedTraceArchitectureArmId, +} from "./fixed-trace-architecture.js"; import { fixedTraceEstimatedCostUsd, validateFixedTracePricing, -} from './fixed-trace-budget.js'; + type FixedTraceBudgetPricing, +} from "./fixed-trace-budget.js"; import { - type FixedTraceCase, - type FixedTracePricing, -} from './fixed-trace-suite.js'; -import { deepFreezeFixedTrace, snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js'; + FIXED_TRACE_PARTITION_MANIFEST, + assertFixedTracePartitionManifest, +} from "./fixed-trace-partition.js"; +import { FIXED_TRACE_CORPUS } from "./fixed-trace-suite.js"; -/** - * A planning-only contract. It has no dispatcher and is deliberately unable - * to make a corpus, an execution envelope, or a confirmatory final pack trusted. - */ export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = - 'addie-fixed-trace-evaluation-protocol-v1' as const; + "addie-fixed-trace-evaluation-protocol-v2" as const; -/** - * Evaluator-owned confirmatory precision rule. The conservative normal - * approximation assumes the maximum possible variance (1) of a paired - * case-level difference in [-1, 1]. The primary family is exactly two - * one-sided claims (superiority and non-inferiority). Holm's first rejection - * is allocated alpha .0125, which yields the conservative normal-approximate - * requirements below. An evaluator-owned exact paired-discordance power - * calculation remains required before a confirmatory claim. - */ export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ - version: 'addie-fixed-trace-confirmatory-power-v1', - unit: 'unique_paired_case', + version: "addie-fixed-trace-confirmatory-power-v2", + familywiseAlpha: 0.025, + hypotheses: Object.freeze([ + Object.freeze({ + id: "H1-superiority", + comparison: "locked-pipeline-candidate vs locked-pipeline-comparator", + endpoint: "two-judge blinded success rate", + direction: "greater", + marginPercentagePoints: 0, + alternativeDifferencePercentagePoints: 5, + holmOneSidedAlpha: 0.0125, + exactTest: "exact_conditional_mcnemar_zero_margin_only", + }), + Object.freeze({ + id: "H2-non-inferiority", + comparison: "locked-pipeline-candidate vs locked-pipeline-comparator", + endpoint: "two-judge blinded success rate", + direction: "not_less_than", + marginPercentagePoints: -3, + alternativeDifferencePercentagePoints: 0, + holmOneSidedAlpha: 0.025, + exactTest: "predeclared_exact_unconditional_matched_pair_test_required", + }), + ]), + test: "exact_paired_discordance_test", + bootstrap: "grouped_stratified_case_level_bootstrap", + exclusionRule: "hard_failures_and_missing_evidence_remain_in_denominator", repetitionsCountAsIndependentCases: false, - primaryHypothesisFamily: Object.freeze({ - size: 2, - correction: 'holm', - orderedOneSidedAlpha: Object.freeze([0.0125, 0.025]), - }), - targetPower: 0.8, - conservativePairedDifferenceVarianceUpperBound: 1, - superiorityMarginPercentagePoints: 5, - nonInferiorityMarginPercentagePoints: -3, superiorityRequiredIndependentEvaluableCases: 3_803, nonInferiorityRequiredIndependentEvaluableCases: 10_562, requiredIndependentEvaluableCases: 10_562, - requiredAnalysis: Object.freeze({ - resampling: 'grouped_stratified_case_level_bootstrap', - multiplicityCorrection: 'holm', - pairedDiscordancePower: 'evaluator_owned_exact_paired_discordance_contract_unavailable', - pairedDiscordanceTest: 'predeclared_exact_paired_test_required', + targetPower: 0.8, + planningAlternative: + "H1: +5pp over zero; H2: 0pp, three points above the -3pp NI margin", + conservativeDiscordanceVarianceUpperBound: 1, + externalFinalN: null, + externalFinalStatus: + "unavailable_pending_fingerprinted_exact_paired_discordance_power_result", +} as const); + +/** + * This is the complete schema of the final statistical admission. Values that + * require independent custody are null, which is an executable refusal—not a + * prose promise. The sizing pilot is held out and may never be reused in the + * one-time final; repeated/template-related observations cluster by episode. + */ +export const FIXED_TRACE_CONFIRMATORY_ADMISSION = Object.freeze({ + status: "not_admitted_missing_fingerprinted_statistical_protocol", + reasons: Object.freeze([ + "external_final_pack_unavailable", + "held_out_sizing_pilot_and_conservative_discordance_bound_unavailable", + "exact_unconditional_noninferiority_test_unavailable", + "candidate_comparator_arm_identity_unavailable", + "judge_calibration_must_be_separate_or_cross_fitted", + ]), + holm: Object.freeze({ + K: 2, + oneSidedFamilyAlpha: 0.025, + orderedAlphas: Object.freeze([0.0125, 0.025]), + }), + unitOfAnalysis: "unique_conversation_user_episode", + repeatedAndTemplateRelatedObservationRule: + "cluster_by_conversation_user_episode; repetitions_never_increase_N", + sizingPilot: Object.freeze({ + status: "unavailable", + heldOutFromFinal: true, + reusableInFinal: false, + conservativeDiscordanceUpperBound: null, + digest: null, + }), + judgeCalibration: Object.freeze({ + status: "unavailable", + allowedRelationshipToScoredDevelopment: "separate_or_cross_fitted_only", + digest: null, }), - currentScreeningTuningUniqueCaseCount: 120, + finalProtocolFingerprint: null, + externalPackDigest: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, } as const); export type FixedTraceProtocolPhaseId = - | 'bounded_smoke' - | 'router_screen' - | 'oracle_generator_ceiling' - | 'deployable_architecture' - | 'controlled_tuning'; - -export type FixedTraceProtocolArchitecture = - | 'two_stage_llm_router' - | 'oracle_route_diagnostic' - | 'hybrid_safe_signal_then_llm' - | 'direct_bounded_production_shaped'; - -export type FixedTraceProtocolStageRole = 'router' | 'generation' | 'judge'; - + | "stage_0_preflight_calibration" + | "stage_1_smoke" + | "stage_2_router_screen" + | "stage_2_oracle_generator_screen" + | "stage_3_architecture" + | "stage_4_tuning" + | "stage_5_external_final" + | "stage_6_canary"; +export type FixedTraceProtocolStageRole = + "router" | "generation" | "judge" | "simulator"; export type FixedTraceProtocolAdmission = - | 'planning_only' - | 'requires_verified_hybrid_contract' - | 'requires_verified_direct_contract'; + | "admitted_diagnostic" + | "not_admitted_architecture" + | "not_evaluable_no_treatment_contrast" + | "not_admitted_external_final" + | "not_admitted_canary"; -export interface FixedTraceProtocolPricingProfile extends FixedTracePricing { - provider: ModelProviderId; - model: string; - /** Immutable price-list revision, distinct from the model identifier. */ - version: string; - /** A plan becomes stale rather than inheriting a later provider price. */ - validBefore: string; +export interface FixedTraceProtocolPricingProfile extends FixedTraceBudgetPricing { + readonly provider: ModelProviderId; + readonly model: string; + readonly version: string; } -/** - * Closed pricing profiles. Cache is disabled in the protocol, but the - * provider-specific semantics remain explicit so a future cache-enabled plan - * must add a reviewed ceiling instead of silently reusing these values. - */ export const FIXED_TRACE_PROTOCOL_PRICING = Object.freeze([ Object.freeze({ - provider: 'openai', - model: OPENAI_ROUTER_MODEL, - profileId: `${OPENAI_GPT_5_6_LUNA_PRICING_VERSION}:${OPENAI_ROUTER_MODEL}`, - version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', - inputUsdPerMillionTokens: 0.2, - outputUsdPerMillionTokens: 1.2, - cacheReadUsdPerMillionTokens: null, - cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'unsupported', - cacheWriteAccounting: 'unsupported', - source: 'Repository OpenAI Luna router price pin, checked 2026-08-26.', - }), - Object.freeze({ - provider: 'anthropic', - model: 'claude-haiku-4-5', - profileId: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, + provider: "anthropic" as const, + model: "claude-haiku-4-5", version: CLAUDE_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', + profileId: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, cacheReadUsdPerMillionTokens: 0.1, cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic standard pricing table, refreshed August 2026.', + cacheReadAccounting: "additive" as const, + cacheWriteAccounting: "additive" as const, + source: "Repository Anthropic reviewed pricing table, August 2026.", }), Object.freeze({ - provider: 'anthropic', - model: 'claude-sonnet-5', - profileId: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, + provider: "anthropic" as const, + model: "claude-sonnet-5", version: CLAUDE_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', + profileId: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, inputUsdPerMillionTokens: 3, outputUsdPerMillionTokens: 15, cacheReadUsdPerMillionTokens: 0.3, cacheWriteUsdPerMillionTokens: 3.75, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic standard pricing table, refreshed August 2026.', + cacheReadAccounting: "additive" as const, + cacheWriteAccounting: "additive" as const, + source: "Repository Anthropic reviewed pricing table, August 2026.", }), Object.freeze({ - provider: 'google', - model: 'gemini-3.7-flash', - profileId: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, + provider: "openai" as const, + model: OPENAI_ROUTER_MODEL, + version: OPENAI_GPT_5_6_LUNA_PRICING.profileId, + ...OPENAI_GPT_5_6_LUNA_PRICING, + }), + Object.freeze({ + provider: "google" as const, + model: GOOGLE_ROUTER_MODEL, version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - validBefore: '2027-01-01T00:00:00.000Z', + profileId: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, inputUsdPerMillionTokens: 0.75, outputUsdPerMillionTokens: 3.75, cacheReadUsdPerMillionTokens: 0.075, cacheWriteUsdPerMillionTokens: 0.75, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'additive', - source: 'Repository Google Gemini 3.7 Flash pricing pin through 2026-12-31.', + cacheReadAccounting: "subset" as const, + cacheWriteAccounting: "additive" as const, + source: + "Repository Google Gemini 3.7 Flash pricing pin through 2026-12-31.", }), ] satisfies readonly FixedTraceProtocolPricingProfile[]); -export interface FixedTraceProtocolStage { - role: FixedTraceProtocolStageRole; - /** Judges receive blinded candidate artifacts; candidate stages do not. */ - blinded: true | null; - provider: ModelProviderId; - model: string; - reasoningEffort: ModelReasoningEffort; - pricingProfileId: string; - /** Hard pre-dispatch cap for one request, not an observed average. */ - maxInputTokensPerInvocation: number; - maxOutputTokensPerInvocation: number; - timeoutMs: number; - maxInvocationsPerCase: number; - transportRetries: 0; - samplingMode: 'provider_no_sampling_control'; - temperature: null; - /** No cache read or write is permitted; profile semantics remain recorded. */ - cacheMode: 'disabled'; +export interface FixedTraceAdmittedCell { + readonly id: string; + readonly role: "router" | "generation"; + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly pricingProfileId: string; + readonly adapterCapabilitySource: string; } +const efforts = (values: readonly ModelReasoningEffort[]) => + values.length ? values : ["provider_default" as const]; +const priceId = (provider: ModelProviderId, model: string) => { + const profile = FIXED_TRACE_PROTOCOL_PRICING.find( + (entry) => entry.provider === provider && entry.model === model, + ); + if (!profile) + throw new Error(`No immutable price for admitted ${provider}/${model}`); + return profile.profileId; +}; +const cells = ( + role: "router" | "generation", + provider: ModelProviderId, + model: string, + values: readonly ModelReasoningEffort[], + source: string, +) => + efforts(values).map((effort) => + Object.freeze({ + id: `${role}:${provider}:${model}:${effort}`, + role, + provider, + model, + effort, + pricingProfileId: priceId(provider, model), + adapterCapabilitySource: source, + }), + ); + +/** Derived only from reviewed exported adapter capabilities and immutable prices. */ +export const FIXED_TRACE_ADMITTED_CELLS: readonly FixedTraceAdmittedCell[] = + Object.freeze([ + ...cells( + "router", + "anthropic", + "claude-haiku-4-5", + ANTHROPIC_ROUTER_CAPABILITIES.reasoningEfforts, + "ANTHROPIC_ROUTER_CAPABILITIES", + ), + ...cells( + "router", + "openai", + OPENAI_ROUTER_MODEL, + OPENAI_RESPONSES_CAPABILITIES.reasoningEfforts, + "OPENAI_RESPONSES_CAPABILITIES", + ), + ...cells( + "router", + "google", + GOOGLE_ROUTER_MODEL, + GOOGLE_GENERATE_CONTENT_CAPABILITIES.reasoningEfforts, + "GOOGLE_GENERATE_CONTENT_CAPABILITIES", + ), + ...cells( + "generation", + "anthropic", + "claude-haiku-4-5", + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts, + "ANTHROPIC_PROVIDER_CAPABILITIES", + ), + ...cells( + "generation", + "anthropic", + "claude-sonnet-5", + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts, + "ANTHROPIC_PROVIDER_CAPABILITIES", + ), + ...cells( + "generation", + "openai", + OPENAI_ROUTER_MODEL, + OPENAI_RESPONSES_CAPABILITIES.reasoningEfforts, + "OPENAI_RESPONSES_CAPABILITIES", + ), + ...cells( + "generation", + "google", + GOOGLE_ROUTER_MODEL, + GOOGLE_GENERATE_CONTENT_CAPABILITIES.reasoningEfforts, + "GOOGLE_GENERATE_CONTENT_CAPABILITIES", + ), + ]); +export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ + Object.freeze({ + provider: "openai", + model: "gpt-5.6-terra", + dispatchable: false, + trustedPrice: null, + }), + Object.freeze({ + provider: "openai", + model: "gpt-5.6-sol", + dispatchable: false, + trustedPrice: null, + }), +]); -const STAGE_FIELD_KEYS = Object.freeze([ - 'role', 'blinded', 'provider', 'model', 'reasoningEffort', 'pricingProfileId', - 'maxInputTokensPerInvocation', 'maxOutputTokensPerInvocation', 'timeoutMs', - 'maxInvocationsPerCase', 'transportRetries', 'samplingMode', 'temperature', 'cacheMode', -] as const); +/** No judge can score a finalist until this evaluator-custodied record exists. */ +export const FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS = Object.freeze([ + Object.freeze({ + provider: "openai" as const, + model: OPENAI_ROUTER_MODEL, + effort: "none" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), + Object.freeze({ + provider: "google" as const, + model: GOOGLE_ROUTER_MODEL, + effort: "provider_default" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), +]); +export interface FixedTraceProtocolStage { + readonly role: FixedTraceProtocolStageRole; + readonly cellId: string | null; + readonly maxInvocationsPerCase: number; + readonly maxInputTokensPerInvocation: number; + readonly maxOutputTokensPerInvocation: number; + readonly timeoutMs: number; + readonly retries: 0; + readonly cacheMode: "disabled"; + readonly sampling: "provider_no_sampling_control"; +} export interface FixedTraceProtocolArm { - id: string; - architecture: FixedTraceProtocolArchitecture; - admission: FixedTraceProtocolAdmission; - /** The three architecture arms share this frozen comparison universe. */ - ablationControlId: string | null; - /** Luna may judge only after an independently verified calibration admission. */ - lunaJudgeCalibration: 'not_applicable' | 'requires_verified_luna_judge_calibration'; - /** Each judge appears once; exactly two are required for compared outputs. */ - stages: readonly FixedTraceProtocolStage[]; + readonly id: string; + readonly architecture: FixedTraceArchitectureArmId | "none"; + readonly admission: FixedTraceProtocolAdmission; + readonly selectedToolSubset: "architecture_derived_presented_subset"; + readonly stages: readonly FixedTraceProtocolStage[]; + readonly conditionalCalls?: { + readonly localTerminalCases: "exact_harmless_only"; + readonly fallbackRouterCallsPerNonlocalCase: 1; + readonly worstCaseRouterCalls: number; + }; } - -/** - * Exactly what the architecture ablation holds fixed. These are contracts for - * a future evaluator, not authority to run one. The generator's stage record - * supplies the exact provider/model/effort/limits; all three final arms use - * that same record. - */ -export const FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL = Object.freeze({ - id: 'fixed-trace-architecture-ablation-v1', - cases: 'same_evaluator_owned_cases_and_order', - generator: 'same_anthropic_claude_sonnet_5_provider_default_stage', - promptToolUniverse: 'same_production_shaped_prompt_system_docs_tools_schemas', - simulatorReceipts: 'same_fixed_trace_simulator_receipts_and_result_provenance', - executionLimits: 'same_input_output_timeout_invocation_retry_cache_sampling_controls', - judging: 'same_two_blinded_provider_excluding_judges', - failureDenominator: 'same_all_planned_case_stage_invocations_including_failures', -} as const); - export interface FixedTraceProtocolPhase { - id: FixedTraceProtocolPhaseId; - uniqueCaseCount: number; - repetitions: number; - /** All output is diagnostic-only and cannot select or promote a candidate. */ - resultUse: 'diagnostic_only'; - arms: readonly FixedTraceProtocolArm[]; + readonly id: FixedTraceProtocolPhaseId; + readonly caseSet: "development" | "tuning" | "external_unavailable"; + readonly uniqueCases: number | null; + readonly repetitions: number; + readonly selectionUse: + | "calibration" + | "adaptive_screening" + | "architecture_selection" + | "diagnostic_tuning" + | "confirmatory_unavailable" + | "default_off_canary_unavailable"; + readonly arms: readonly FixedTraceProtocolArm[]; } - export interface FixedTraceEvaluationProtocol { - version: typeof FIXED_TRACE_EVALUATION_PROTOCOL_VERSION; - id: string; - /** This identifier must be resolved by a future evaluator-owned coordinator. */ - trustedManifestId: string; - pricingAsOf: string; - contingencyBasisPoints: number; - /** A planning deficit only; it is not an executable or authenticated phase. */ - unavailableFinalTarget: { - availability: 'unavailable'; - uniqueCaseCount: number; - repetitions: number; - missingCaseCount: number; + readonly version: typeof FIXED_TRACE_EVALUATION_PROTOCOL_VERSION; + readonly id: string; + readonly baseCapabilityUniverse: "one_authenticated_base_registry_schema_receipt_set"; + readonly phases: readonly FixedTraceProtocolPhase[]; + readonly adaptiveRule: { + readonly smokeCases: 8; + readonly developmentCases: 46; + readonly tuningCases: 36; + readonly deterministicElimination: readonly string[]; + readonly selection: "predeclared_pareto_successive_halving"; + readonly repeats: "stability_only_not_new_cases"; }; - phases: readonly FixedTraceProtocolPhase[]; -} - -export interface FixedTraceProtocolTrustedManifest { - id: string; - protocolFingerprint: string; - sourceId: string; - sourceRevision: string; - /** - * Evaluator-owned digests of the actual subsets passed to the runner. They - * are not canonical-suite constants and must be supplied as the repaired - * runner's `traceSuite` and `traceSuiteSha256` config before dispatch; - * post-hoc observation restamping is forbidden. - */ - traceSuiteSha256ByPhase: Readonly>; - tracePackSha256: string; - rawLedgerVersion: string; - partitions: Readonly>; - verifiedAdmissions: readonly FixedTraceProtocolAdmission[]; -} - -export type FixedTraceProtocolTrustedManifestResolver = - (id: string) => FixedTraceProtocolTrustedManifest | null; - -/** - * The only suite-identity input a future dispatcher may pass to the repaired - * runner. It is derived from evaluator-owned state before dispatch, never - * inferred from or applied to a completed observation. - */ -export interface FixedTraceProtocolRunnerBinding { - trustedManifestId: string; - protocolFingerprint: string; - phaseId: FixedTraceProtocolPhaseId; - /** Evaluator-owned subset, passed unchanged to the repaired runner. */ - traceSuite: ReadonlyArray; - /** Matches the repaired runner's required `traceSuiteSha256` config field. */ - traceSuiteSha256: string; -} - -export interface FixedTraceProtocolStageEstimate { - phaseId: FixedTraceProtocolPhaseId; - armId: string; - role: FixedTraceProtocolStageRole; - provider: ModelProviderId; - model: string; - reasoningEffort: ModelReasoningEffort; - pricingProfileId: string; - cacheMode: 'disabled'; - cacheSemantics: Pick; - requests: number; - inputTokenCeiling: number; - outputTokenCeiling: number; - ceilingUsd: number; -} - -export interface FixedTraceProtocolPhaseEstimate { - phaseId: FixedTraceProtocolPhaseId; - uniqueCaseCount: number; - repetitions: number; - candidateCalls: number; - judgeCalls: number; - candidateCeilingUsd: number; - judgeCeilingUsd: number; - totalCeilingUsd: number; -} - -export interface FixedTraceProtocolEstimate { - protocolFingerprint: string; - dispatchable: false; - expectedSpendUsd: null; - stages: readonly FixedTraceProtocolStageEstimate[]; - phases: readonly FixedTraceProtocolPhaseEstimate[]; - screening: { candidateCeilingUsd: number; judgeCeilingUsd: number; contingencyUsd: number; totalCeilingUsd: number }; - unavailableFinalTarget: FixedTraceEvaluationProtocol['unavailableFinalTarget']; - /** The confirmatory sample remains unpriced and cannot authorize spend. */ - budgetProjection: { - screeningTuning: { - uniqueEvaluableCaseCount: number; - repetitionsCountAsIndependentCases: false; - expectedSpendUsd: null; - approvalCeilingUsd: null; - }; - confirmatory: { - requiredIndependentEvaluableCaseCount: number; - unavailableTargetCaseCount: number; - expectedSpendUsd: null; - approvalCeilingUsd: null; - spendAuthorization: 'refused_pending_evaluator_owned_paired_test'; - }; + readonly finalProtocol: { + readonly status: "unavailable"; + readonly familywiseAlpha: 0.025; + readonly hypothesisIds: readonly ["H1-superiority", "H2-non-inferiority"]; + readonly endpoint: "two-judge blinded success rate"; + readonly externalPackDigest: null; + readonly externalN: null; + readonly candidatePipelineId: null; + readonly comparatorPipelineId: null; + readonly architectureArmId: null; + readonly pairedTest: "exact_paired_discordance_test"; + readonly bootstrap: "grouped_stratified_case_level_bootstrap"; + readonly exclusions: "hard_failures_and_missing_evidence_remain_in_denominator"; + readonly fingerprint: null; + readonly powerResult: null; }; - candidateCeilingUsd: number; - judgeCeilingUsd: number; - contingencyUsd: number; - totalCeilingUsd: number; } -export interface FixedTraceConfirmatoryClaimInput { - /** One entry per observed paired evaluation; repeated IDs remain one case. */ - pairedCaseIds: readonly string[]; - observedSuperiorityPercentagePoints: number; - observedNonInferiorityPercentagePoints: number; -} +const stage = ( + role: FixedTraceProtocolStageRole, + cellId: string | null, + maxInvocationsPerCase: number, + maxInputTokensPerInvocation: number, + maxOutputTokensPerInvocation: number, +): FixedTraceProtocolStage => + Object.freeze({ + role, + cellId, + maxInvocationsPerCase, + maxInputTokensPerInvocation, + maxOutputTokensPerInvocation, + timeoutMs: 120_000, + retries: 0, + cacheMode: "disabled", + sampling: "provider_no_sampling_control", + }); +const routerCell = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "router:anthropic:claude-haiku-4-5:provider_default", +)!; +const generatorCell = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:anthropic:claude-sonnet-5:provider_default", +)!; +const judgeCells = Object.freeze([ + FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:openai:gpt-5.6-luna:none", + )!, + FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:google:gemini-3.7-flash:provider_default", + )!, +]); +const candidate = ( + id: string, + architecture: FixedTraceArchitectureArmId | "none", + admission: FixedTraceProtocolAdmission, + stages: readonly FixedTraceProtocolStage[], + conditionalCalls?: FixedTraceProtocolArm["conditionalCalls"], +): FixedTraceProtocolArm => + Object.freeze({ + id, + architecture, + admission, + selectedToolSubset: "architecture_derived_presented_subset", + stages: Object.freeze(stages), + ...(conditionalCalls ? { conditionalCalls } : {}), + }); -export interface FixedTraceConfirmatoryClaimGate { - independentEvaluableCaseCount: number; - repeatedObservationCount: number; - requiredIndependentEvaluableCaseCount: number; - nominalMarginsReached: boolean; - confirmatoryClaim: 'refused_underpowered' | 'refused_pending_evaluator_owned_paired_test'; +export interface FixedTraceHybridContrastPreflight { + readonly phase: "development" | "tuning"; + readonly totalCases: number; + readonly localTerminalCases: number; + readonly routedCases: number; + readonly minimumLocalTerminalCases: 1; + readonly minimumRoutedCases: 1; + readonly evaluable: boolean; + readonly blocker: "no_hybrid_treatment_contrast" | null; } /** - * Counts only distinct paired case IDs. Crossing a nominal quality margin is - * descriptive until the evaluator has both the predeclared sample and its - * grouped/stratified case-level bootstrap, Holm correction, and exact paired - * discordance test. This offline planner can never promote a candidate. + * Positivity is measured only against the advertised corpus. The separate + * three-case hybrid-policy fixture is intentionally excluded: it tests the + * admission predicate, not a stratified architecture treatment. */ -export function evaluateFixedTraceConfirmatoryClaim( - input: FixedTraceConfirmatoryClaimInput, -): FixedTraceConfirmatoryClaimGate { - const snapshot = snapshotFixedTraceJson(input, 'confirmatory claim') as FixedTraceConfirmatoryClaimInput; - assertExactKeys(snapshot, [ - 'pairedCaseIds', 'observedSuperiorityPercentagePoints', 'observedNonInferiorityPercentagePoints', - ], 'confirmatory claim'); - const pairedCaseIds = snapshot.pairedCaseIds; - if (pairedCaseIds.some((caseId) => typeof caseId !== 'string' || !caseId.trim())) { - throw new Error('Confirmatory paired case IDs must be nonblank strings'); - } - if (!Number.isFinite(snapshot.observedSuperiorityPercentagePoints) - || !Number.isFinite(snapshot.observedNonInferiorityPercentagePoints)) { - throw new Error('Confirmatory observed margins must be finite'); - } - const independentEvaluableCaseCount = new Set(pairedCaseIds).size; - const repeatedObservationCount = pairedCaseIds.length - independentEvaluableCaseCount; - const nominalMarginsReached = snapshot.observedSuperiorityPercentagePoints - >= FIXED_TRACE_CONFIRMATORY_POWER_GATE.superiorityMarginPercentagePoints - && snapshot.observedNonInferiorityPercentagePoints - >= FIXED_TRACE_CONFIRMATORY_POWER_GATE.nonInferiorityMarginPercentagePoints; +export function fixedTraceHybridContrastPreflight( + phase: "development" | "tuning", +): FixedTraceHybridContrastPreflight { + const traces = FIXED_TRACE_CORPUS.filter((trace) => trace.phase === phase); + const localTerminalCases = traces.filter( + (trace) => + decideFixedTraceHybridRoute({ + message: trace.request.message, + source: trace.request.source, + isAdmin: trace.request.isAdmin, + isThread: (trace.request.threadContext?.length ?? 0) > 0, + channelPrivacy: trace.request.channelPrivacy, + policy: fixedTraceHybridPolicy(), + }).mode === "local_terminal", + ).length; + const routedCases = traces.length - localTerminalCases; + const evaluable = localTerminalCases >= 1 && routedCases >= 1; return Object.freeze({ - independentEvaluableCaseCount, - repeatedObservationCount, - requiredIndependentEvaluableCaseCount: FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases, - nominalMarginsReached, - confirmatoryClaim: independentEvaluableCaseCount - < FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases - ? 'refused_underpowered' - : 'refused_pending_evaluator_owned_paired_test', + phase, + totalCases: traces.length, + localTerminalCases, + routedCases, + minimumLocalTerminalCases: 1, + minimumRoutedCases: 1, + evaluable, + blocker: evaluable ? null : "no_hybrid_treatment_contrast", }); } -function canonicalJson(value: unknown): string { - if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); - if (typeof value === 'number') { - if (!Number.isFinite(value)) throw new Error('Protocol contains a non-finite number'); - return JSON.stringify(value); - } - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; - if (typeof value === 'object') { - const record = value as Record; - return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; - } - throw new Error('Protocol contains a non-JSON value'); -} - -function sha256(value: unknown): string { - return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); -} - -function positiveInteger(value: number, label: string): void { - if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`); -} - -function assertExactKeys(value: object, keys: readonly string[], label: string): void { - const actual = Object.keys(value).sort(); - if (actual.some((key) => key === '__proto__' || key === 'prototype' || key === 'constructor')) { - throw new Error(`${label} contains a dangerous prototype key`); - } - const expected = [...keys].sort(); - if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { - throw new Error(`${label} has unknown, missing, or inherited fields`); - } -} - -function pricing(profileId: string, pricingAsOf: string): FixedTraceProtocolPricingProfile { - const profile = FIXED_TRACE_PROTOCOL_PRICING.find((candidate) => candidate.profileId === profileId); - if (!profile) throw new Error(`Unavailable immutable pricing profile: ${profileId}`); - const asOf = new Date(pricingAsOf); - if (Number.isNaN(asOf.getTime()) || asOf >= new Date(profile.validBefore)) { - throw new Error(`Stale immutable pricing profile: ${profileId}`); - } - validateFixedTracePricing(profile); - return profile; -} - -function assertStage(stage: FixedTraceProtocolStage, label: string, pricingAsOf: string): FixedTraceProtocolPricingProfile { - assertExactKeys(stage, STAGE_FIELD_KEYS, label); - positiveInteger(stage.maxInputTokensPerInvocation, `${label}.maxInputTokensPerInvocation`); - positiveInteger(stage.maxOutputTokensPerInvocation, `${label}.maxOutputTokensPerInvocation`); - positiveInteger(stage.timeoutMs, `${label}.timeoutMs`); - positiveInteger(stage.maxInvocationsPerCase, `${label}.maxInvocationsPerCase`); - if (stage.transportRetries !== 0 || stage.samplingMode !== 'provider_no_sampling_control' || stage.temperature !== null || stage.cacheMode !== 'disabled') { - throw new Error(`${label} has an unsupported execution control`); - } - if ((stage.role === 'judge' && stage.blinded !== true) || (stage.role !== 'judge' && stage.blinded !== null)) { - throw new Error(`${label} has an invalid blinded-judge control`); - } - const resolved = pricing(stage.pricingProfileId, pricingAsOf); - if ( - resolved.profileId !== stage.pricingProfileId - || resolved.provider !== stage.provider - || resolved.model !== stage.model - ) throw new Error(`${label} pricing profile does not match its requested provider/model`); - return resolved; -} - -/** - * Execution limits are evaluator-owned planning inputs, not caller-selected - * estimates. Keep this matrix independent of the proposed protocol object so - * a detached protocol supplied to an offline estimator cannot rewrite its - * phase, arm, admission, result-use, or execution configuration. The private - * stage records are the only allowed price-bearing configurations; callers - * may provide JSON that equals them, but cannot select a price cohort. - */ -const PRICE = Object.freeze({ - luna: `${OPENAI_GPT_5_6_LUNA_PRICING_VERSION}:${OPENAI_ROUTER_MODEL}`, - haiku: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, - sonnet: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, - gemini: `${GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION}:gemini-3.7-flash`, -}); - -const router = ( - provider: ModelProviderId, - model: string, - reasoningEffort: ModelReasoningEffort, - pricingProfileId: string, -): FixedTraceProtocolStage => Object.freeze({ - role: 'router', blinded: null, provider, model, reasoningEffort, pricingProfileId, - maxInputTokensPerInvocation: 4_096, maxOutputTokensPerInvocation: 300, - timeoutMs: 120_000, maxInvocationsPerCase: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', -}); - -const generation = ( - provider: ModelProviderId, - model: string, - reasoningEffort: ModelReasoningEffort, - pricingProfileId: string, -): FixedTraceProtocolStage => Object.freeze({ - role: 'generation', blinded: null, provider, model, reasoningEffort, pricingProfileId, - maxInputTokensPerInvocation: 16_384, maxOutputTokensPerInvocation: 900, - timeoutMs: 120_000, maxInvocationsPerCase: 12, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', -}); +export const FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT = Object.freeze([ + fixedTraceHybridContrastPreflight("development"), + fixedTraceHybridContrastPreflight("tuning"), +]); -const judge = ( - provider: ModelProviderId, - model: string, - reasoningEffort: ModelReasoningEffort, - pricingProfileId: string, -): FixedTraceProtocolStage => Object.freeze({ - role: 'judge', blinded: true, provider, model, reasoningEffort, pricingProfileId, - maxInputTokensPerInvocation: 16_384, maxOutputTokensPerInvocation: 300, - timeoutMs: 120_000, maxInvocationsPerCase: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', +export const FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL = Object.freeze({ + id: "fixed-trace-architecture-ablation-v2", + fixed: Object.freeze([ + "cases_and_order", + "locked_generator_finalist", + "one_authenticated_base_registry_schema_receipt_set", + "rules_prompts_simulator_receipts", + "limits_retries_cache_sampling", + "two_calibrated_blinded_provider_excluding_judges", + "all_planned_failures_denominator", + ]), + varied: "architecture_derived_tool_selection_and_presented_subset_only", }); -const NO_ABLATION = null; -const NO_LUNA_JUDGE_CALIBRATION = 'not_applicable' as const; -const LUNA_JUDGE_CALIBRATION = 'requires_verified_luna_judge_calibration' as const; -const ABLATION = FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id; - -const EVALUATOR_OWNED_PHASE_MATRIX = Object.freeze([ +export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = Object.freeze({ - id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, - arms: Object.freeze([Object.freeze({ - id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), + version: FIXED_TRACE_EVALUATION_PROTOCOL_VERSION, + id: "addie-fixed-trace-adaptive-plan-v2", + baseCapabilityUniverse: + "one_authenticated_base_registry_schema_receipt_set", + adaptiveRule: Object.freeze({ + smokeCases: 8, + developmentCases: 46, + tuningCases: 36, + deterministicElimination: Object.freeze([ + "identity_or_pricing_mismatch", + "unauthorized_or_incorrect_mutation", + "malformed_empty_or_truncated_output", + "tool_loop_or_iteration_boundary", + "timeout_or_provider_error", + "missing_usage_or_ledger_mismatch", + "privacy_violation", ]), - })]), - }), - Object.freeze({ - id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, - arms: Object.freeze([Object.freeze({ - id: 'router-haiku-default', architecture: 'two_stage_llm_router', admission: 'planning_only', - ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]), - }), Object.freeze({ - id: 'router-luna-none', architecture: 'two_stage_llm_router', admission: 'planning_only', - ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([router('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna)]), - })]), - }), - Object.freeze({ - id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, - arms: Object.freeze([Object.freeze({ - id: 'generator-sonnet-default', architecture: 'oracle_route_diagnostic', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet)]), - }), Object.freeze({ - id: 'generator-haiku-default', architecture: 'oracle_route_diagnostic', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([generation('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]), - })]), - }), - Object.freeze({ - id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, - arms: Object.freeze([ + selection: "predeclared_pareto_successive_halving", + repeats: "stability_only_not_new_cases", + }), + finalProtocol: Object.freeze({ + status: "unavailable", + familywiseAlpha: 0.025, + hypothesisIds: Object.freeze([ + "H1-superiority", + "H2-non-inferiority", + ]) as readonly ["H1-superiority", "H2-non-inferiority"], + endpoint: "two-judge blinded success rate", + externalPackDigest: null, + externalN: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, + pairedTest: "exact_paired_discordance_test", + bootstrap: "grouped_stratified_case_level_bootstrap", + exclusions: "hard_failures_and_missing_evidence_remain_in_denominator", + fingerprint: null, + powerResult: null, + }), + phases: Object.freeze([ + Object.freeze({ + id: "stage_0_preflight_calibration", + caseSet: "development", + uniqueCases: 8, + repetitions: 1, + selectionUse: "calibration", + arms: Object.freeze([]), + }), + Object.freeze({ + id: "stage_1_smoke", + caseSet: "development", + uniqueCases: 8, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.map((cell) => + candidate(`smoke-${cell.id}`, "none", "admitted_diagnostic", [ + stage( + cell.role, + cell.id, + cell.role === "router" ? 1 : 12, + cell.role === "router" ? 4_096 : 16_384, + cell.role === "router" ? 300 : 900, + ), + ]), + ), + ), + }), + Object.freeze({ + id: "stage_2_router_screen", + caseSet: "development", + uniqueCases: 46, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.role === "router", + ).map((cell) => + candidate( + `router-screen-${cell.id}`, + "two_stage_llm_router", + "admitted_diagnostic", + [stage("router", cell.id, 1, 4_096, 300)], + ), + ), + ), + }), Object.freeze({ - id: 'routed-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), - judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), + id: "stage_2_oracle_generator_screen", + caseSet: "development", + uniqueCases: 46, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.role === "generation", + ).map((cell) => + candidate( + `generator-screen-${cell.id}`, + "oracle_route_diagnostic", + "admitted_diagnostic", + [stage("generation", cell.id, 12, 16_384, 900)], + ), + ), + ), + }), + Object.freeze({ + id: "stage_3_architecture", + caseSet: "development", + uniqueCases: 46, + repetitions: 3, + selectionUse: "architecture_selection", + arms: Object.freeze([ + candidate( + "routed-locked-finalist", + "two_stage_llm_router", + "admitted_diagnostic", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + stage("judge", judgeCells[0].id, 1, 16_384, 300), + stage("judge", judgeCells[1].id, 1, 16_384, 300), + ], + ), + candidate( + "hybrid-locked-finalist", + "deterministic_policy_llm_fallback_hybrid", + "not_evaluable_no_treatment_contrast", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + stage("judge", judgeCells[0].id, 1, 16_384, 300), + stage("judge", judgeCells[1].id, 1, 16_384, 300), + ], + { + localTerminalCases: "exact_harmless_only", + fallbackRouterCallsPerNonlocalCase: 1, + worstCaseRouterCalls: 46 * 3, + }, + ), + candidate( + "direct-locked-finalist", + "direct_generation", + "not_admitted_architecture", + [stage("generation", generatorCell.id, 12, 16_384, 900)], + ), + ]), + }), + Object.freeze({ + id: "stage_4_tuning", + caseSet: "tuning", + uniqueCases: 36, + repetitions: 1, + selectionUse: "diagnostic_tuning", + arms: Object.freeze([ + candidate( + "tuning-locked-pipeline", + "two_stage_llm_router", + "admitted_diagnostic", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + ], + ), ]), }), Object.freeze({ - id: 'safe-hybrid-sonnet', architecture: 'hybrid_safe_signal_then_llm', admission: 'requires_verified_hybrid_contract', ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([ - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), - judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), + id: "stage_5_external_final", + caseSet: "external_unavailable", + uniqueCases: null, + repetitions: 1, + selectionUse: "confirmatory_unavailable", + arms: Object.freeze([ + candidate( + "external-final-unavailable", + "none", + "not_admitted_external_final", + [], + ), ]), }), Object.freeze({ - id: 'bounded-direct-sonnet', architecture: 'direct_bounded_production_shaped', admission: 'requires_verified_direct_contract', ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([ - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), - judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), + id: "stage_6_canary", + caseSet: "external_unavailable", + uniqueCases: null, + repetitions: 1, + selectionUse: "default_off_canary_unavailable", + arms: Object.freeze([ + candidate("canary-unavailable", "none", "not_admitted_canary", []), ]), }), ]), - }), - Object.freeze({ - id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, - arms: Object.freeze([Object.freeze({ - id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - ]), - })]), - }), -] as const); + }); -function matchesEvaluatorOwnedStage( - stage: FixedTraceProtocolStage, - expected: FixedTraceProtocolStage, -): boolean { - return stage.role === expected.role - && stage.blinded === expected.blinded - && stage.provider === expected.provider - && stage.model === expected.model - && stage.reasoningEffort === expected.reasoningEffort - && stage.pricingProfileId === expected.pricingProfileId - && stage.maxInputTokensPerInvocation === expected.maxInputTokensPerInvocation - && stage.maxOutputTokensPerInvocation === expected.maxOutputTokensPerInvocation - && stage.timeoutMs === expected.timeoutMs - && stage.maxInvocationsPerCase === expected.maxInvocationsPerCase - && stage.transportRetries === expected.transportRetries - && stage.samplingMode === expected.samplingMode - && stage.temperature === expected.temperature - && stage.cacheMode === expected.cacheMode; +export interface FixedTraceStageCeiling { + phaseId: FixedTraceProtocolPhaseId; + armId: string; + role: FixedTraceProtocolStageRole; + calls: number; + ceilingUsd: number; } - -function assertEvaluatorOwnedPhaseMatrix(phase: FixedTraceProtocolPhase, index: number): void { - const expected = EVALUATOR_OWNED_PHASE_MATRIX[index]; - if (!expected - || phase.id !== expected.id - || phase.uniqueCaseCount !== expected.uniqueCaseCount - || phase.repetitions !== expected.repetitions - || phase.resultUse !== 'diagnostic_only') { - throw new Error('Protocol phase does not match the evaluator-owned phase matrix'); - } - if (phase.arms.length !== expected.arms.length) { - throw new Error(`${phase.id} arms do not match the evaluator-owned phase matrix`); - } - for (let armIndex = 0; armIndex < phase.arms.length; armIndex += 1) { - const arm = phase.arms[armIndex]; - const expectedArm = expected.arms[armIndex]; - if (!expectedArm - || arm.id !== expectedArm.id - || arm.architecture !== expectedArm.architecture - || arm.admission !== expectedArm.admission - || arm.ablationControlId !== expectedArm.ablationControlId - || arm.lunaJudgeCalibration !== expectedArm.lunaJudgeCalibration) { - throw new Error(`${phase.id} arm does not match the evaluator-owned arm matrix`); - } - if (arm.stages.length !== expectedArm.stages.length) { - throw new Error(`${phase.id}.${arm.id} does not match the evaluator-owned stage configuration matrix`); - } - for (let stageIndex = 0; stageIndex < arm.stages.length; stageIndex += 1) { - const stage = arm.stages[stageIndex]; - const expectedStage = expectedArm.stages[stageIndex]; - assertExactKeys(stage, STAGE_FIELD_KEYS, `${phase.id}.${arm.id}.stage[${stageIndex}]`); - if (!expectedStage || !matchesEvaluatorOwnedStage(stage, expectedStage)) { - throw new Error(`${phase.id}.${arm.id} does not match the evaluator-owned stage configuration matrix`); - } - } - } +export interface FixedTraceProtocolEstimate { + dispatchable: false; + approvalCeilingUsd: null; + stages: readonly FixedTraceStageCeiling[]; + candidateCeilingUsd: number; + judgeCeilingUsd: number; + simulatorCeilingUsd: 0; + failedTimeoutUnknownExposureCeilingUsd: number; + contingencyUsd: number; + totalCeilingUsd: number; + hybridWorstCaseRouterCalls: 138; + hybridWorstCaseRouterCeilingUsd: number; + armCallAccounting: readonly FixedTraceArchitectureArmCallAccounting[]; + externalFinalN: null; } - -function assertArm(phase: FixedTraceProtocolPhase, arm: FixedTraceProtocolArm, pricingAsOf: string): void { - assertExactKeys(arm, ['id', 'architecture', 'admission', 'ablationControlId', 'lunaJudgeCalibration', 'stages'], `protocol arm`); - if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid protocol arm ID: ${arm.id}`); - const routers = arm.stages.filter((stage) => stage.role === 'router'); - const generations = arm.stages.filter((stage) => stage.role === 'generation'); - const judges = arm.stages.filter((stage) => stage.role === 'judge'); - if (phase.id === 'router_screen') { - if (arm.architecture !== 'two_stage_llm_router' || routers.length !== 1 || generations.length !== 0 || judges.length !== 0) { - throw new Error(`${arm.id} is not a router-only screening arm`); - } - assertStage(routers[0], `${arm.id}.router`, pricingAsOf); - return; - } - if (generations.length !== 1 || routers.length > 1) throw new Error(`${arm.id} requires exactly one generation stage and at most one router`); - if (phase.id === 'deployable_architecture') { - if (arm.ablationControlId !== ABLATION) throw new Error(`${arm.id} must use the fixed architecture ablation control`); - const candidateProviders = new Set([...routers, ...generations].map((stage) => stage.provider)); - if (candidateProviders.size !== 1) throw new Error(`${arm.id} must be a single-provider candidate pipeline for independent judging`); - if (arm.architecture === 'two_stage_llm_router' && (routers.length !== 1 || arm.admission !== 'planning_only')) { - throw new Error(`${arm.id} must use its locked routed architecture contract`); - } - if (arm.architecture === 'hybrid_safe_signal_then_llm' && (routers.length !== 0 || arm.admission !== 'requires_verified_hybrid_contract')) { - throw new Error(`${arm.id} requires its verified hybrid admission`); - } - if (arm.architecture === 'direct_bounded_production_shaped' && (routers.length !== 0 || arm.admission !== 'requires_verified_direct_contract')) { - throw new Error(`${arm.id} requires its verified direct admission`); - } - if (!['two_stage_llm_router', 'hybrid_safe_signal_then_llm', 'direct_bounded_production_shaped'].includes(arm.architecture)) { - throw new Error(`${arm.id} is not an architecture-ablation candidate`); - } - if (judges.length !== 2 || arm.stages.slice(-2).some((stage) => stage.role !== 'judge')) { - throw new Error(`${arm.id} requires exactly two trailing blinded judges`); - } - const judgeProviders = new Set(judges.map((stage) => stage.provider)); - if (judgeProviders.size !== 2 || [...judgeProviders].some((provider) => candidateProviders.has(provider))) { - throw new Error(`${arm.id} judges must be provider-excluding and independent`); - } - const usesLunaJudge = judges.some((stage) => stage.provider === 'openai' && stage.model === OPENAI_ROUTER_MODEL); - if (usesLunaJudge !== (arm.lunaJudgeCalibration === LUNA_JUDGE_CALIBRATION)) { - throw new Error(`${arm.id} Luna judge calibration admission is not locked`); - } - } else if (arm.ablationControlId !== NO_ABLATION || arm.lunaJudgeCalibration !== NO_LUNA_JUDGE_CALIBRATION) { - throw new Error(`${arm.id} has architecture-ablation controls outside the ablation phase`); - } else if (arm.architecture === 'two_stage_llm_router') { - if (routers.length !== 1) throw new Error(`${arm.id} requires a router stage`); - } else if (arm.architecture !== 'oracle_route_diagnostic' || routers.length !== 0) { - throw new Error(`${arm.id} direct and hybrid substitutions are not admitted`); - } - if (arm.architecture === 'oracle_route_diagnostic' && phase.id !== 'oracle_generator_ceiling') { - throw new Error(`${arm.id} oracle routing is diagnostic-only`); - } - for (const stage of arm.stages) assertStage(stage, `${arm.id}.${stage.role}`, pricingAsOf); - if (phase.id !== 'deployable_architecture' && judges.length !== 0) throw new Error(`${arm.id} judges are blocked outside the architecture ablation`); +export interface FixedTraceArchitectureArmCallAccounting { + readonly armId: string; + readonly admission: FixedTraceProtocolAdmission; + readonly evaluable: boolean; + readonly localTerminalCases: number; + readonly routedCases: number; + readonly routerCalls: number; + readonly generationCalls: number; + readonly routerCeilingUsd: number; + readonly generationCeilingUsd: number; } - -function validatedProtocolSnapshot(protocol: FixedTraceEvaluationProtocol): FixedTraceEvaluationProtocol { - const snapshot = snapshotFixedTraceJson(protocol, 'evaluation protocol') as FixedTraceEvaluationProtocol; - assertFixedTraceEvaluationProtocolStructure(snapshot); - return snapshot; +export interface FixedTraceScreeningResult { + readonly cellId: string; + readonly safetyFailures: number; + readonly identityFailures: number; + readonly malformedFailures: number; + readonly toolLoopFailures: number; + readonly reliabilityFailures: number; + readonly latencyMs: number; + readonly costUsd: number; } - -/** Fingerprints the exact detached projection which passed all protocol checks. */ -export function fixedTraceEvaluationProtocolFingerprint(protocol: FixedTraceEvaluationProtocol): string { - return sha256(validatedProtocolSnapshot(protocol)); -} - -/** Validate the planning projection without loading traces, credentials, or providers. */ -function assertFixedTraceEvaluationProtocolStructure(protocol: FixedTraceEvaluationProtocol): void { - assertExactKeys(protocol, [ - 'version', 'id', 'trustedManifestId', 'pricingAsOf', 'contingencyBasisPoints', - 'unavailableFinalTarget', 'phases', - ], 'evaluation protocol'); - if (protocol.version !== FIXED_TRACE_EVALUATION_PROTOCOL_VERSION || !protocol.id.trim() || !protocol.trustedManifestId.trim()) { - throw new Error('Unsupported or incomplete fixed-trace evaluation protocol'); - } - if (!Number.isSafeInteger(protocol.contingencyBasisPoints) || protocol.contingencyBasisPoints < 0 || protocol.contingencyBasisPoints > 10_000) { - throw new Error('Protocol contingency basis points are invalid'); - } - const phaseIds = new Set(); - const armIds = new Set(); - assertExactKeys(protocol.unavailableFinalTarget, ['availability', 'uniqueCaseCount', 'repetitions', 'missingCaseCount'], 'evaluation protocol.unavailableFinalTarget'); - if (protocol.unavailableFinalTarget.availability !== 'unavailable' - || protocol.unavailableFinalTarget.uniqueCaseCount !== 38 - || protocol.unavailableFinalTarget.repetitions !== 3 - || protocol.unavailableFinalTarget.missingCaseCount !== 38) { - throw new Error('Protocol unavailable final target is invalid'); - } - if (protocol.phases.length !== EVALUATOR_OWNED_PHASE_MATRIX.length - || protocol.phases.some((phase, index) => phase.id !== EVALUATOR_OWNED_PHASE_MATRIX[index]?.id)) { - throw new Error('Protocol phases must use the exact required order'); - } - for (const [index, phase] of protocol.phases.entries()) { - assertExactKeys(phase, ['id', 'uniqueCaseCount', 'repetitions', 'resultUse', 'arms'], 'protocol phase'); - if (phaseIds.has(phase.id)) throw new Error(`Duplicate protocol phase: ${phase.id}`); - phaseIds.add(phase.id); - assertEvaluatorOwnedPhaseMatrix(phase, index); - for (const arm of phase.arms) { - if (armIds.has(arm.id)) throw new Error(`Duplicate protocol arm ID: ${arm.id}`); - armIds.add(arm.id); - assertArm(phase, arm, protocol.pricingAsOf); - } - } - for (const required of EVALUATOR_OWNED_PHASE_MATRIX) { - if (!phaseIds.has(required.id)) throw new Error(`Protocol is missing required phase: ${required.id}`); +/** Pure, predeclared elimination/halving rule; repetitions estimate stability only. */ +export function selectFixedTraceScreeningSurvivors( + results: readonly FixedTraceScreeningResult[], +): readonly string[] { + const seen = new Set(); + for (const result of results) { + if ( + !FIXED_TRACE_ADMITTED_CELLS.some((cell) => cell.id === result.cellId) || + seen.has(result.cellId) + ) + throw new Error("screening result has an unknown or duplicate cell"); + if ( + [ + result.safetyFailures, + result.identityFailures, + result.malformedFailures, + result.toolLoopFailures, + result.reliabilityFailures, + result.latencyMs, + result.costUsd, + ].some((value) => !Number.isFinite(value) || value < 0) + ) + throw new Error("screening result has invalid metrics"); + seen.add(result.cellId); } + const eligible = results.filter( + (result) => + result.safetyFailures === 0 && + result.identityFailures === 0 && + result.malformedFailures === 0 && + result.toolLoopFailures === 0, + ); + return Object.freeze( + [...eligible] + .sort( + (left, right) => + left.reliabilityFailures - right.reliabilityFailures || + left.costUsd - right.costUsd || + left.latencyMs - right.latencyMs || + left.cellId.localeCompare(right.cellId), + ) + .slice(0, Math.max(1, Math.ceil(eligible.length / 2))) + .map((result) => result.cellId), + ); } - -export function assertFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): void { - void validatedProtocolSnapshot(protocol); +function sha256(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(value), "utf8") + .digest("hex"); } - -/** - * Future execution must supply evaluator-owned data. This check intentionally - * does not make a JSON protocol file trusted by comparing it to itself. - */ -export function assertFixedTraceEvaluationProtocolTrusted( +export function fixedTraceEvaluationProtocolFingerprint( protocol: FixedTraceEvaluationProtocol, - resolver: FixedTraceProtocolTrustedManifestResolver, -): FixedTraceProtocolTrustedManifest { - void protocol; - void resolver; - throw new Error('Trusted evaluation manifest is locked pending evaluator-owned authentication'); +): string { + assertFixedTraceEvaluationProtocol(protocol); + return sha256(protocol); } - -export function fixedTraceEvaluationProtocolRunnerBinding( +export function assertFixedTraceEvaluationProtocol( protocol: FixedTraceEvaluationProtocol, - resolver: FixedTraceProtocolTrustedManifestResolver, - phaseId: FixedTraceProtocolPhaseId, - traceSuite: readonly FixedTraceCase[], -): FixedTraceProtocolRunnerBinding { - void protocol; - void resolver; - void phaseId; - void traceSuite; - throw new Error('Fixed-trace execution is locked pending evaluator-owned authentication'); -} - -function stageEstimate( - phase: FixedTraceProtocolPhase, - arm: FixedTraceProtocolArm, - stage: FixedTraceProtocolStage, - pricingAsOf: string, -): FixedTraceProtocolStageEstimate { - const profile = assertStage(stage, `${phase.id}.${arm.id}.${stage.role}`, pricingAsOf); - const requests = phase.uniqueCaseCount * phase.repetitions * stage.maxInvocationsPerCase; - const inputTokenCeiling = requests * stage.maxInputTokensPerInvocation; - const outputTokenCeiling = requests * stage.maxOutputTokensPerInvocation; - const ceilingUsd = fixedTraceEstimatedCostUsd({ - inputTokens: inputTokenCeiling, - outputTokens: outputTokenCeiling, - cacheReadTokens: 0, - cacheWriteTokens: 0, - }, profile); - return Object.freeze({ - phaseId: phase.id, - armId: arm.id, - role: stage.role, - provider: stage.provider, - model: stage.model, - reasoningEffort: stage.reasoningEffort, - pricingProfileId: profile.profileId, - cacheMode: stage.cacheMode, - cacheSemantics: Object.freeze({ - cacheReadAccounting: profile.cacheReadAccounting, - cacheWriteAccounting: profile.cacheWriteAccounting, - }), - requests, - inputTokenCeiling, - outputTokenCeiling, - ceilingUsd, - }); +): void { + assertFixedTracePartitionManifest(); + if ( + protocol.version !== FIXED_TRACE_EVALUATION_PROTOCOL_VERSION || + protocol.baseCapabilityUniverse !== + "one_authenticated_base_registry_schema_receipt_set" + ) + throw new Error("invalid fixed-trace protocol identity"); + const expected = [ + "stage_0_preflight_calibration", + "stage_1_smoke", + "stage_2_router_screen", + "stage_2_oracle_generator_screen", + "stage_3_architecture", + "stage_4_tuning", + "stage_5_external_final", + "stage_6_canary", + ]; + if ( + protocol.phases.length !== expected.length || + protocol.phases.some((phase, index) => phase.id !== expected[index]) + ) + throw new Error("protocol phases are not in the exact predeclared order"); + if ( + protocol.finalProtocol.status !== "unavailable" || + protocol.finalProtocol.externalN !== null || + protocol.finalProtocol.externalPackDigest !== null || + protocol.finalProtocol.candidatePipelineId !== null || + protocol.finalProtocol.comparatorPipelineId !== null || + protocol.finalProtocol.architectureArmId !== null || + protocol.finalProtocol.fingerprint !== null || + protocol.finalProtocol.powerResult !== null + ) + throw new Error( + "external final is unavailable until exact paired-discordance power is fingerprinted", + ); + if ( + FIXED_TRACE_CONFIRMATORY_ADMISSION.status !== + "not_admitted_missing_fingerprinted_statistical_protocol" || + FIXED_TRACE_CONFIRMATORY_ADMISSION.finalProtocolFingerprint !== null || + FIXED_TRACE_CONFIRMATORY_ADMISSION.sizingPilot.digest !== null || + FIXED_TRACE_CONFIRMATORY_ADMISSION.judgeCalibration.digest !== null + ) { + throw new Error( + "confirmatory statistical admission must fail closed until independently custodied", + ); + } + for (const phase of protocol.phases) { + const expectedCases = + phase.caseSet === "development" + ? FIXED_TRACE_PARTITION_MANIFEST.development.length + : phase.caseSet === "tuning" + ? FIXED_TRACE_PARTITION_MANIFEST.tuning.length + : null; + if ( + phase.uniqueCases !== null && + phase.uniqueCases !== 8 && + phase.uniqueCases !== expectedCases + ) + throw new Error( + `phase ${phase.id} does not use corpus-derived case counts`, + ); + for (const arm of phase.arms) { + if ( + arm.architecture === "direct_generation" && + arm.admission !== "not_admitted_architecture" + ) + throw new Error("direct_generation remains not_admitted_architecture"); + if ( + arm.architecture === "deterministic_policy_llm_fallback_hybrid" && + (!arm.conditionalCalls || + !arm.stages.some((item) => item.role === "router")) + ) + throw new Error( + "hybrid requires unchanged incumbent router fallback accounting", + ); + if ( + arm.architecture === "deterministic_policy_llm_fallback_hybrid" && + !FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT.find( + (preflight) => preflight.phase === "development", + )!.evaluable && + arm.admission !== "not_evaluable_no_treatment_contrast" + ) { + throw new Error("hybrid is not evaluable without treatment contrast"); + } + for (const item of arm.stages) + if ( + item.cellId !== null && + !FIXED_TRACE_ADMITTED_CELLS.some((cell) => cell.id === item.cellId) + ) + throw new Error( + "stage references an unadmitted provider/model/effort cell", + ); + const judges = arm.stages.filter((item) => item.role === "judge"); + if (judges.length) { + const expectedJudges = assertPromotionGradeDualJudgeFeasibility( + arm, + ).map((cell) => cell.id); + if ( + judges.length !== 2 || + judges + .map((item) => item.cellId) + .some((id, index) => id !== expectedJudges[index]) + ) + throw new Error( + "semantic judges must be the two calibrated providers excluding every pipeline provider", + ); + } + } + } } - -/** - * Pure deterministic diagnostic projection. It makes no provider calls, reads - * no trace body, and writes no output. `expectedSpendUsd` stays null because - * observed tokenization and tool-loop length are deliberately not guessed. - */ -export function estimateFixedTraceEvaluationProtocol(protocol: FixedTraceEvaluationProtocol): FixedTraceProtocolEstimate { - const snapshot = validatedProtocolSnapshot(protocol); - const stages = snapshot.phases.flatMap((phase) => phase.arms.flatMap((arm) => - arm.stages.map((stage) => stageEstimate(phase, arm, stage, snapshot.pricingAsOf)))); - const phases = snapshot.phases.map((phase) => { - const entries = stages.filter((entry) => entry.phaseId === phase.id); - const candidate = entries.filter((entry) => entry.role !== 'judge'); - const judges = entries.filter((entry) => entry.role === 'judge'); - const candidateCeilingUsd = candidate.reduce((total, entry) => total + entry.ceilingUsd, 0); - const judgeCeilingUsd = judges.reduce((total, entry) => total + entry.ceilingUsd, 0); +export function estimateFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, +): FixedTraceProtocolEstimate { + assertFixedTraceEvaluationProtocol(protocol); + const stages: FixedTraceStageCeiling[] = []; + for (const phase of protocol.phases) + for (const arm of phase.arms) { + if ( + (arm.admission === "not_admitted_architecture" || + phase.uniqueCases === null) && + arm.architecture !== "deterministic_policy_llm_fallback_hybrid" + ) + continue; + const uniqueCases = phase.uniqueCases; + if (uniqueCases === null) continue; + for (const item of arm.stages) { + const cell = FIXED_TRACE_ADMITTED_CELLS.find( + (entry) => entry.id === item.cellId, + ); + if (!cell) continue; + const profile = FIXED_TRACE_PROTOCOL_PRICING.find( + (entry) => entry.profileId === cell.pricingProfileId, + )!; + validateFixedTracePricing(profile); + const calls = + uniqueCases * phase.repetitions * item.maxInvocationsPerCase; + stages.push({ + phaseId: phase.id, + armId: arm.id, + role: item.role, + calls, + ceilingUsd: fixedTraceEstimatedCostUsd( + { + inputTokens: calls * item.maxInputTokensPerInvocation, + outputTokens: calls * item.maxOutputTokensPerInvocation, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + profile, + ), + }); + } + } + const candidateCeilingUsd = stages + .filter((item) => item.role === "router" || item.role === "generation") + .reduce((total, item) => total + item.ceilingUsd, 0); + const judgeCeilingUsd = stages + .filter((item) => item.role === "judge") + .reduce((total, item) => total + item.ceilingUsd, 0); + const failedTimeoutUnknownExposureCeilingUsd = + candidateCeilingUsd + judgeCeilingUsd; + const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * 0.1; + const hybridRouter = stages.find( + (item) => + item.phaseId === "stage_3_architecture" && + item.armId === "hybrid-locked-finalist" && + item.role === "router", + )!; + const architecturePhase = protocol.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + const developmentContrast = FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT.find( + (preflight) => preflight.phase === "development", + )!; + const armCallAccounting = architecturePhase.arms.map((arm) => { + const router = arm.stages.find((item) => item.role === "router"); + const generation = arm.stages.find((item) => item.role === "generation"); + const localTerminalCases = + arm.architecture === "deterministic_policy_llm_fallback_hybrid" + ? developmentContrast.localTerminalCases * architecturePhase.repetitions + : 0; + const routedCases = + arm.architecture === "direct_generation" + ? 0 + : architecturePhase.uniqueCases! * architecturePhase.repetitions - + localTerminalCases; + const cost = (item: FixedTraceProtocolStage | undefined, calls: number) => { + if (!item?.cellId) return 0; + const cell = FIXED_TRACE_ADMITTED_CELLS.find( + (entry) => entry.id === item.cellId, + )!; + const profile = FIXED_TRACE_PROTOCOL_PRICING.find( + (entry) => entry.profileId === cell.pricingProfileId, + )!; + return fixedTraceEstimatedCostUsd( + { + inputTokens: calls * item.maxInputTokensPerInvocation, + outputTokens: calls * item.maxOutputTokensPerInvocation, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + profile, + ); + }; + const routerCalls = router ? routedCases * router.maxInvocationsPerCase : 0; + const generationCalls = generation + ? routedCases * generation.maxInvocationsPerCase + : 0; return Object.freeze({ - phaseId: phase.id, - uniqueCaseCount: phase.uniqueCaseCount, - repetitions: phase.repetitions, - candidateCalls: candidate.reduce((total, entry) => total + entry.requests, 0), - judgeCalls: judges.reduce((total, entry) => total + entry.requests, 0), - candidateCeilingUsd, - judgeCeilingUsd, - totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd, + armId: arm.id, + admission: arm.admission, + evaluable: + arm.architecture !== "deterministic_policy_llm_fallback_hybrid" || + developmentContrast.evaluable, + localTerminalCases, + routedCases, + routerCalls, + generationCalls, + routerCeilingUsd: cost(router, routerCalls), + generationCeilingUsd: cost(generation, generationCalls), }); }); - const candidateCeilingUsd = phases.reduce((total, phase) => total + phase.candidateCeilingUsd, 0); - const judgeCeilingUsd = phases.reduce((total, phase) => total + phase.judgeCeilingUsd, 0); - const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * snapshot.contingencyBasisPoints / 10_000; - const summarize = (source: readonly FixedTraceProtocolPhaseEstimate[]) => { - const candidateCeilingUsd = source.reduce((total, phase) => total + phase.candidateCeilingUsd, 0); - const judgeCeilingUsd = source.reduce((total, phase) => total + phase.judgeCeilingUsd, 0); - const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * snapshot.contingencyBasisPoints / 10_000; - return Object.freeze({ candidateCeilingUsd, judgeCeilingUsd, contingencyUsd, totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd + contingencyUsd }); - }; return Object.freeze({ - protocolFingerprint: sha256(snapshot), dispatchable: false, - expectedSpendUsd: null, + approvalCeilingUsd: null, stages: Object.freeze(stages), - phases: Object.freeze(phases), - screening: summarize(phases), - unavailableFinalTarget: snapshot.unavailableFinalTarget, - budgetProjection: Object.freeze({ - screeningTuning: Object.freeze({ - uniqueEvaluableCaseCount: FIXED_TRACE_CONFIRMATORY_POWER_GATE.currentScreeningTuningUniqueCaseCount, - repetitionsCountAsIndependentCases: false, - expectedSpendUsd: null, - approvalCeilingUsd: null, - }), - confirmatory: Object.freeze({ - requiredIndependentEvaluableCaseCount: FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases, - unavailableTargetCaseCount: snapshot.unavailableFinalTarget.uniqueCaseCount, - expectedSpendUsd: null, - approvalCeilingUsd: null, - spendAuthorization: 'refused_pending_evaluator_owned_paired_test', - }), - }), candidateCeilingUsd, judgeCeilingUsd, + simulatorCeilingUsd: 0, + failedTimeoutUnknownExposureCeilingUsd, contingencyUsd, totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd + contingencyUsd, + hybridWorstCaseRouterCalls: 138, + hybridWorstCaseRouterCeilingUsd: hybridRouter.ceilingUsd, + armCallAccounting: Object.freeze(armCallAccounting), + externalFinalN: null, }); } - -/** Unsupported model names are inert metadata, never a stage or a price. */ -export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ - Object.freeze({ provider: 'openai' as const, model: 'gpt-5.6-terra', dispatchable: false as const, trustedPrice: null }), - Object.freeze({ provider: 'openai' as const, model: 'gpt-5.6-sol', dispatchable: false as const, trustedPrice: null }), -]); - -/** A closed, diagnostic-only projection with no promotion or execution path. */ -export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = deepFreezeFixedTrace({ - version: FIXED_TRACE_EVALUATION_PROTOCOL_VERSION, - id: 'addie-6842-6846-staged-v1', - trustedManifestId: 'externally-owned-addie-fixed-trace-v120', - pricingAsOf: '2026-09-05T12:00:00.000Z', - // Includes explicit failure/usage accounting contingency in every reported - // screening ceiling; it is still a non-authorizing offline projection. - contingencyBasisPoints: 2_000, - unavailableFinalTarget: { - availability: 'unavailable', uniqueCaseCount: 38, repetitions: 3, missingCaseCount: 38, - }, - phases: Object.freeze([ - Object.freeze({ - id: 'bounded_smoke', uniqueCaseCount: 8, repetitions: 1, resultUse: 'diagnostic_only', - arms: Object.freeze([Object.freeze({ - id: 'smoke-incumbent-two-stage', architecture: 'two_stage_llm_router', admission: 'planning_only', ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, - stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - ]), - })]), - }), - Object.freeze({ - id: 'router_screen', uniqueCaseCount: 46, repetitions: 3, resultUse: 'diagnostic_only', - arms: Object.freeze([ - Object.freeze({ id: 'router-haiku-default', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]) }), - Object.freeze({ id: 'router-luna-none', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([router('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna)]) }), - ]), - }), - Object.freeze({ - id: 'oracle_generator_ceiling', uniqueCaseCount: 46, repetitions: 2, resultUse: 'diagnostic_only', - arms: Object.freeze([ - Object.freeze({ id: 'generator-sonnet-default', architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet)]) }), - Object.freeze({ id: 'generator-haiku-default', architecture: 'oracle_route_diagnostic' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([generation('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku)]) }), - ]), - }), - Object.freeze({ - id: 'deployable_architecture', uniqueCaseCount: 46, repetitions: 3, resultUse: 'diagnostic_only', - arms: Object.freeze([ - Object.freeze({ id: 'routed-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), - judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), - ]) }), - Object.freeze({ id: 'safe-hybrid-sonnet', architecture: 'hybrid_safe_signal_then_llm' as const, admission: 'requires_verified_hybrid_contract' as const, ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), - judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), - ]) }), - Object.freeze({ id: 'bounded-direct-sonnet', architecture: 'direct_bounded_production_shaped' as const, admission: 'requires_verified_direct_contract' as const, ablationControlId: ABLATION, lunaJudgeCalibration: LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ - generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - judge('openai', OPENAI_ROUTER_MODEL, 'none', PRICE.luna), - judge('google', 'gemini-3.7-flash', 'provider_default', PRICE.gemini), - ]) }), - ]), - }), - Object.freeze({ - id: 'controlled_tuning', uniqueCaseCount: 36, repetitions: 3, resultUse: 'diagnostic_only', - arms: Object.freeze([ - Object.freeze({ id: 'tuning-incumbent-haiku-sonnet', architecture: 'two_stage_llm_router' as const, admission: 'planning_only' as const, ablationControlId: NO_ABLATION, lunaJudgeCalibration: NO_LUNA_JUDGE_CALIBRATION, stages: Object.freeze([ - router('anthropic', 'claude-haiku-4-5', 'provider_default', PRICE.haiku), generation('anthropic', 'claude-sonnet-5', 'provider_default', PRICE.sonnet), - ]) }), - ]), - }), - ]), -}); +/** Promotion-grade semantic inference excludes every LLM used in the pipeline. */ +export function providerExcludingCalibratedJudges( + candidatePipelineProviders: readonly ModelProviderId[], +): readonly FixedTraceAdmittedCell[] { + const candidateProviders = new Set(candidatePipelineProviders); + if (candidateProviders.size !== 1) { + throw new Error( + "promotion-grade dual-LLM judging requires a single-provider complete pipeline; mixed finalists require a human-primary path or fourth calibrated provider", + ); + } + return Object.freeze( + (["anthropic", "openai", "google"] as const) + .filter((provider) => !candidateProviders.has(provider)) + .map((provider) => + FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => + cell.role === "generation" && + cell.provider === provider && + (provider === "openai" + ? cell.effort === "none" + : cell.effort === "provider_default"), + )!, + ), + ); +} +export function semanticJudgeCandidateProviders( + arm: Pick, +): readonly ModelProviderId[] { + const providers = arm.stages + .filter((stage) => stage.role === "router" || stage.role === "generation") + .map((stage) => + FIXED_TRACE_ADMITTED_CELLS.find((cell) => cell.id === stage.cellId), + ); + if ( + !providers.some((cell) => cell && cell.role === "generation") || + providers.some((cell) => !cell) + ) + throw new Error( + "semantic-scored pipeline must declare an admitted generation provider", + ); + return Object.freeze([...new Set(providers.map((cell) => cell!.provider))]); +} +export function assertPromotionGradeDualJudgeFeasibility( + arm: Pick, +): readonly FixedTraceAdmittedCell[] { + return providerExcludingCalibratedJudges( + semanticJudgeCandidateProviders(arm), + ); +} diff --git a/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts new file mode 100644 index 0000000000..6898f77cbd --- /dev/null +++ b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts @@ -0,0 +1,386 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import type { + ModelProviderId, + ModelReasoningEffort, +} from "../model-providers/model-provider.js"; + +/** + * Evaluator-owned custody for offline fixed-trace evidence. This module has no + * provider client and no production-handler import. Its signing key is passed + * only by the evaluator's protected configuration; candidate plans, artifacts, + * and callbacks cannot mint, alter, or restamp an expected sequence. + */ +export const FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION = + "addie-fixed-trace-evaluator-coordinator-v1" as const; + +export type FixedTraceLedgerTamperClass = + | "omission" + | "insertion" + | "duplication" + | "substitution" + | "reordering" + | "authentication" + | "unknown_exposure"; +export class FixedTraceLedgerValidationError extends Error { + constructor( + readonly tamperClass: FixedTraceLedgerTamperClass, + message: string, + ) { + super(message); + this.name = "FixedTraceLedgerValidationError"; + } +} + +export interface FixedTraceExpectedInvocation { + readonly runId: string; + readonly phaseId: string; + readonly caseId: string; + readonly armId: string; + readonly stage: "router" | "generation" | "judge" | "simulator"; + readonly invocation: number; + readonly attempt: number; + readonly requested: { + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly identityPolicy: string; + }; + readonly controls: { + readonly promptSha256: string; + readonly systemSha256: string; + readonly messagesSha256: string; + readonly toolSchemaSha256: string; + readonly providerRequestSha256: string; + readonly presentedToolNames: readonly string[]; + readonly presentedToolOrderSha256: string; + readonly simulatorReceiptProvenanceSha256: string; + readonly simulatorControlsSha256: string; + readonly architectureSha256: string; + readonly admissionSha256: string; + readonly configSha256: string; + readonly pricingSha256: string; + readonly limitsSha256: string; + readonly retryCacheSamplingSha256: string; + readonly failureDenominatorId: string; + }; +} +export interface FixedTraceActualInvocation extends FixedTraceExpectedInvocation { + readonly returned: { + readonly provider: ModelProviderId | null; + readonly model: string | null; + readonly identityPolicy: string | null; + }; + readonly toolCallsSha256: string | null; + readonly toolInputsSha256: string | null; + readonly toolResultsSha256: string | null; + readonly startedAt: string; + readonly finishedAt: string; + readonly latencyMs: number; + readonly usage: { + readonly inputTokens: number; + readonly outputTokens: number; + readonly cacheReadTokens: number; + readonly cacheWriteTokens: number; + } | null; + readonly pricing: { + readonly profileId: string; + readonly costUsd: number; + } | null; + readonly terminalStatus: + | "complete" + | "timeout_after_dispatch" + | "provider_error" + | "malformed" + | "empty" + | "truncated" + | "tool_boundary" + | "privacy_violation" + | "not_dispatched_budget" + | "unknown_exposure"; + readonly errorCode: string | null; +} +export interface FixedTraceExpectedSequenceContract { + readonly version: typeof FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION; + readonly runId: string; + readonly protocolFingerprint: string; + readonly manifestFingerprint: string; + readonly entries: readonly FixedTraceExpectedInvocation[]; + readonly signature: string; +} +export interface FixedTraceEvidenceLedger { + readonly contract: FixedTraceExpectedSequenceContract; + readonly entries: readonly FixedTraceActualInvocation[]; + readonly complete: boolean; + readonly halted: boolean; + readonly plannedDenominator: number; + readonly observedDenominator: number; + readonly hardFailureDenominator: number; +} + +function canonical(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new Error("non-finite evaluator ledger value"); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`) + .join(",")}}`; + } + throw new Error("non-JSON evaluator ledger value"); +} +const invocationKey = ( + entry: Pick< + FixedTraceExpectedInvocation, + | "runId" + | "phaseId" + | "caseId" + | "armId" + | "stage" + | "invocation" + | "attempt" + >, +) => + [ + entry.runId, + entry.phaseId, + entry.caseId, + entry.armId, + entry.stage, + entry.invocation, + entry.attempt, + ].join("\u0000"); +const contractProjection = ( + contract: Omit, +) => canonical(contract); +const sameExpected = ( + actual: FixedTraceActualInvocation, + expected: FixedTraceExpectedInvocation, +) => { + const { + returned, + toolCallsSha256, + toolInputsSha256, + toolResultsSha256, + startedAt, + finishedAt, + latencyMs, + usage, + pricing, + terminalStatus, + errorCode, + ...requested + } = actual; + void returned; + void toolCallsSha256; + void toolInputsSha256; + void toolResultsSha256; + void startedAt; + void finishedAt; + void latencyMs; + void usage; + void pricing; + void terminalStatus; + void errorCode; + return canonical(requested) === canonical(expected); +}; + +export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { + readonly hmacKey: Uint8Array; + readonly keyId: string; +}) { + if ( + !(evaluatorConfig.hmacKey instanceof Uint8Array) || + evaluatorConfig.hmacKey.byteLength < 32 || + !evaluatorConfig.keyId.trim() + ) + throw new Error("evaluator-owned HMAC custody configuration is required"); + const sign = (projection: string) => + createHmac("sha256", evaluatorConfig.hmacKey) + .update( + `${FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION}\u0000${evaluatorConfig.keyId}\u0000${projection}`, + ) + .digest("hex"); + const verify = (contract: FixedTraceExpectedSequenceContract) => { + const projection = contractProjection({ + version: contract.version, + runId: contract.runId, + protocolFingerprint: contract.protocolFingerprint, + manifestFingerprint: contract.manifestFingerprint, + entries: contract.entries, + }); + const expected = Buffer.from(sign(projection), "hex"); + const supplied = Buffer.from(contract.signature, "hex"); + if ( + contract.version !== FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION || + expected.length !== supplied.length || + !timingSafeEqual(expected, supplied) + ) + throw new FixedTraceLedgerValidationError( + "authentication", + "expected sequence contract authentication failed", + ); + }; + return Object.freeze({ + issueExpectedSequence( + input: Omit, + ): FixedTraceExpectedSequenceContract { + if ( + !input.runId.trim() || + !input.protocolFingerprint.trim() || + !input.manifestFingerprint.trim() || + input.entries.length === 0 + ) + throw new Error( + "complete evaluator-owned expected sequence is required before dispatch", + ); + const keys = new Set(); + for (const entry of input.entries) { + const key = invocationKey(entry); + if (entry.runId !== input.runId || keys.has(key)) + throw new Error( + "expected sequence has a duplicate or wrong-run invocation", + ); + keys.add(key); + } + const unsigned = { + version: FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION, + ...input, + } as const; + return Object.freeze({ + ...unsigned, + entries: Object.freeze([...input.entries]), + signature: sign(contractProjection(unsigned)), + }); + }, + validate( + contract: FixedTraceExpectedSequenceContract, + actualEntries: readonly FixedTraceActualInvocation[], + ): FixedTraceEvidenceLedger { + verify(contract); + const observed: FixedTraceActualInvocation[] = []; + const seen = new Set(); + let halted = false; + for (const actual of actualEntries) { + if (halted) + throw new FixedTraceLedgerValidationError( + "unknown_exposure", + "run was halted after unknown exposure", + ); + const key = invocationKey(actual); + const expected = contract.entries[observed.length]; + const knownIndex = contract.entries.findIndex( + (entry) => invocationKey(entry) === key, + ); + if (seen.has(key)) + throw new FixedTraceLedgerValidationError( + "duplication", + "ledger duplicated an invocation", + ); + if (knownIndex < 0) + throw new FixedTraceLedgerValidationError( + "insertion", + "ledger inserted an unplanned invocation", + ); + if (!expected) + throw new FixedTraceLedgerValidationError( + "insertion", + "ledger exceeded the pre-dispatch expected sequence", + ); + if (knownIndex > observed.length) { + const expectedKey = invocationKey(expected); + const appearsLater = actualEntries + .slice(observed.length + 1) + .some((entry) => invocationKey(entry) === expectedKey); + throw new FixedTraceLedgerValidationError( + appearsLater ? "reordering" : "omission", + appearsLater + ? "ledger reordered an expected invocation" + : "ledger omitted an expected invocation before a later one", + ); + } + if (knownIndex < observed.length) + throw new FixedTraceLedgerValidationError( + "reordering", + "ledger reordered an expected invocation", + ); + if (!sameExpected(actual, expected)) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger substituted evaluator-owned requested identity or controls", + ); + if ( + !Number.isFinite(actual.latencyMs) || + actual.latencyMs < 0 || + Number.isNaN(Date.parse(actual.startedAt)) || + Number.isNaN(Date.parse(actual.finishedAt)) + ) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger has invalid timing evidence", + ); + if (Date.parse(actual.finishedAt) < Date.parse(actual.startedAt)) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger finished before it started", + ); + if (actual.terminalStatus === "unknown_exposure") { + halted = true; + throw new FixedTraceLedgerValidationError( + "unknown_exposure", + "unknown provider exposure halts the run", + ); + } + if (actual.terminalStatus !== "not_dispatched_budget") { + if ( + actual.returned.provider !== expected.requested.provider || + actual.returned.model !== expected.requested.model || + actual.returned.identityPolicy !== expected.requested.identityPolicy + ) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger returned a different provider, model, or identity policy", + ); + if ( + !actual.usage || + !actual.pricing || + !Number.isFinite(actual.pricing.costUsd) || + actual.pricing.costUsd < 0 || + Object.values(actual.usage).some( + (value) => !Number.isSafeInteger(value) || value < 0, + ) + ) + throw new FixedTraceLedgerValidationError( + "substitution", + "dispatched invocation lacks complete trusted usage or pricing", + ); + } + seen.add(key); + observed.push(actual); + } + if (observed.length !== contract.entries.length) + throw new FixedTraceLedgerValidationError( + "omission", + "ledger ended before its planned denominator", + ); + const hardFailureDenominator = observed.filter( + (entry) => entry.terminalStatus !== "complete", + ).length; + return Object.freeze({ + contract, + entries: Object.freeze(observed), + complete: true, + halted: false, + plannedDenominator: contract.entries.length, + observedDenominator: observed.length, + hardFailureDenominator, + }); + }, + }); +} diff --git a/server/src/addie/eval/fixed-trace-experiment-plan.ts b/server/src/addie/eval/fixed-trace-experiment-plan.ts deleted file mode 100644 index 1368e5b134..0000000000 --- a/server/src/addie/eval/fixed-trace-experiment-plan.ts +++ /dev/null @@ -1,719 +0,0 @@ -import { createHash } from 'node:crypto'; -import type { ModelProviderId, ModelReasoningEffort } from '../model-providers/model-provider.js'; -import { - GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - OPENAI_GPT_5_6_LUNA_PRICING_VERSION, -} from '../model-cost-pricing.js'; -import { CLAUDE_PRICING_VERSION } from '../claude-pricing.js'; -import { OPENAI_ROUTER_MODEL } from '../model-providers/openai-responses-provider.js'; -import { - FIXED_TRACE_PARTITION_MANIFEST, - FIXED_TRACE_PARTITION_MANIFEST_SHA256, - FIXED_TRACE_PARTITION_MANIFEST_VERSION, - assertFixedTracePartitionManifest, -} from './fixed-trace-partition.js'; -import type { AddieTool } from '../types.js'; -import { CODE_VERSION } from '../config-version.js'; -import { - FIXED_TRACE_STAGE_CONTROL_VERSION, - type FixedTraceCase, -} from './fixed-trace-suite.js'; -import type { FixedTraceToolDefinitionProvenance } from './fixed-trace-architecture.js'; -import { deepFreezeFixedTrace, snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js'; - -/** A versioned, network-free admission contract for fixed-trace experiments. */ -export const FIXED_TRACE_EXPERIMENT_PLAN_VERSION = 'addie-fixed-trace-experiment-plan-v1' as const; -export const FIXED_TRACE_RAW_LEDGER_VERSION = 'addie-fixed-trace-raw-ledger-v1' as const; -export const FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION = - 'repository_visible_development_validation_not_confirmatory_holdout' as const; - -export type FixedTraceExperimentArchitecture = - | 'two_stage_llm_router' - | 'direct_generation' - | 'hybrid_generation' - | 'oracle_route_diagnostic'; -export type FixedTraceScreeningStage = - | 'router_only_screen' - | 'oracle_route_generator_diagnostic' - | 'deployable_finalist'; - -export interface FixedTraceImmutablePricingProfile { - provider: ModelProviderId; - model: string; - version: string; - validBefore: string; - inputUsdPerMillionTokens: number; - outputUsdPerMillionTokens: number; - source: string; -} - -/** - * Profiles are deliberately a closed list. A missing provider/model/version is - * unavailable, rather than inheriting a sibling model's price. - */ -export const FIXED_TRACE_IMMUTABLE_PRICING = Object.freeze([ - Object.freeze({ - provider: 'openai', model: OPENAI_ROUTER_MODEL, version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, - source: 'Repository OpenAI Luna router price pin, checked 2026-08-26.', - }), - Object.freeze({ - provider: 'anthropic', model: 'claude-haiku-4-5', version: CLAUDE_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, - source: 'Repository Anthropic standard pricing table, refreshed August 2026.', - }), - Object.freeze({ - provider: 'anthropic', model: 'claude-sonnet-5', version: CLAUDE_PRICING_VERSION, - validBefore: '2026-09-06T00:00:00.000Z', inputUsdPerMillionTokens: 3, outputUsdPerMillionTokens: 15, - source: 'Repository Anthropic standard pricing table, refreshed August 2026.', - }), - Object.freeze({ - provider: 'google', model: 'gemini-3.7-flash', version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - validBefore: '2027-01-01T00:00:00.000Z', inputUsdPerMillionTokens: 0.75, outputUsdPerMillionTokens: 3.75, - source: 'Repository Google Gemini 3.7 Flash pricing pin through 2026-12-31.', - }), -] satisfies readonly FixedTraceImmutablePricingProfile[]); - -export interface FixedTraceRequestBounds { - /** One exact UTF-8 request byte count for every possible request in the loop. */ - inputBytesByTrace: Readonly>; -} - -export interface FixedTracePlannedStage { - provider: ModelProviderId; - model: string; - reasoningEffort: ModelReasoningEffort; - pricingVersion: string; - maxOutputTokens: number; - timeoutMs: number; - maxIterations: number; - transportRetries: 0; - samplingMode: 'temperature_zero' | 'provider_no_sampling_control'; - temperature: 0 | null; - /** Cache accounting is unavailable for execution unless disabled. */ - cacheMode: 'disabled'; - requestBounds: FixedTraceRequestBounds; -} - -export interface FixedTracePlannedJudge extends FixedTracePlannedStage { - blinded: true; -} - -export interface FixedTraceExperimentArm { - id: string; - architecture: FixedTraceExperimentArchitecture; - screeningStage: FixedTraceScreeningStage; - repetitionIndex: number; - router?: FixedTracePlannedStage; - generation?: FixedTracePlannedStage; - judges?: readonly FixedTracePlannedJudge[]; -} - -export interface FixedTraceExperimentPlan { - version: typeof FIXED_TRACE_EXPERIMENT_PLAN_VERSION; - id: string; - /** Resolved outside the candidate-controlled plan before it is admissible. */ - trustedManifestId: string; - sourceId: string; - sourceRevision: string; - pricingAsOf: string; - sourceBundleSha256: string; - /** Exact values stamped by the runner and included in its provenance hash. */ - gitCommit: string; - gitDirty: boolean; - addieCodeVersion: string; - stageControlVersion: string; - traceSuiteSha256: string; - promptConfigVersion: string; - toolSchemaSha256: string; - toolDefinitionProvenance: FixedTraceToolDefinitionProvenance; - providerDegradationInjectionEnabled: boolean; - partition: { - manifestVersion: typeof FIXED_TRACE_PARTITION_MANIFEST_VERSION; - manifestSha256: typeof FIXED_TRACE_PARTITION_MANIFEST_SHA256; - selected: 'development' | 'repository_visible_development_validation'; - }; - ordering: { seed: string }; - budgets: { candidateCeilingUsd: number; judgeCeilingUsd: number }; - arms: readonly FixedTraceExperimentArm[]; -} - -/** - * The resolver is deliberately external to the plan file. A plan cannot make - * itself trusted by repeating its own hashes. The future dispatcher must use - * an attested/controlled resolver, never deserialize this alongside a plan. - */ -export interface FixedTraceTrustedManifest { - id: string; - sourceId: string; - sourceRevision: string; - sourceBundleSha256: string; - promptConfigVersion: string; - /** - * Resolver-owned, phase-selected execution inputs. They are never inferred - * from a plan or copied onto an observation after execution. - */ - suites: Readonly>; - partitionManifestSha256: string; - rawLedgerVersion: typeof FIXED_TRACE_RAW_LEDGER_VERSION; - gitCommit: string; - gitDirty: boolean; - addieCodeVersion: string; - stageControlVersion: string; - providerDegradationInjectionEnabled: boolean; -} - -export interface FixedTraceTrustedSuite { - traceSuite: ReadonlyArray; - traceSuiteSha256: string; - toolDefinitions: ReadonlyArray; - toolSchemaSha256: string; - toolDefinitionProvenance: FixedTraceToolDefinitionProvenance; -} - -export type FixedTraceTrustedManifestResolver = (id: string) => FixedTraceTrustedManifest | null; - -/** Stable identity for a resolver-owned manifest, used by the raw ledger. */ -export function fixedTraceTrustedManifestFingerprint(manifest: FixedTraceTrustedManifest): string { - return sha256(snapshotFixedTraceJson(manifest, 'trusted manifest')); -} - -/** - * The evaluator-owned portion of a future runner config. A dispatcher must - * supply actual providers separately, but may not replace any value here or - * synthesize a suite/hash from a completed observation. - */ -export interface FixedTraceExperimentRunnerBinding { - runId: string; - repetition: number; - sourceBundleSha256: string; - gitCommit: string; - gitDirty: boolean; - addieCodeVersion: string; - stageControlVersion: string; - promptConfigVersion: string; - traceSuite: ReadonlyArray; - traceSuiteSha256: string; - toolDefinitions: ReadonlyArray; - toolDefinitionProvenance: FixedTraceToolDefinitionProvenance; - providerDegradationInjectionEnabled: boolean; -} - -/** - * A confirmatory finalization authority is intentionally absent. The only - * in-repository secondary split is development validation, not a secret pack; - * an externally authored/custodied final pack needs its own reviewed system. - */ - -/** - * An untrusted proposed observation shape. It cannot be validated or used as - * evidence in this PR. A future evaluator must bind every field below to a - * `FixedTraceTrustedExecutionExpectation` before the matching invocation. - */ -export interface FixedTraceRawLedgerEntry { - sequence: number; - /** The plan-controlled stage grouping; it cannot be supplied out of order. */ - phaseId: FixedTraceScreeningStage; - armId: string; - repetitionIndex: number; - traceId: string; - stage: 'router' | 'generation' | 'judge'; - /** Monotonic configured invocation within a stage; never collapsed to one row. */ - callIndex: number; - /** Every network attempt is represented, including failed attempts. */ - attemptIndex: number; - dispatched: boolean; - requestedProvider: ModelProviderId | null; - requestedModel: string | null; - returnedProvider: ModelProviderId | null; - returnedModel: string | null; - promptSha256: string; - systemSha256: string; - docsSha256: string; - toolSchemaSha256: string; - providerRequestSha256: string | null; - responseSha256: string | null; - /** Content-addressed immutable raw artifacts; their bytes stay outside summaries. */ - rawRequestArtifact: { sha256: string; byteLength: number; storageKey: string } | null; - rawResponseArtifact: { sha256: string; byteLength: number; storageKey: string } | null; - exactToolNames: readonly string[]; - caseControlSha256: string; - executionEnvelopeSha256: string; - directAdmissionSha256: string; - simulatorReceiptSha256: string; - simulatorResultProvenanceSha256: string; - maxOutputTokens: number | null; - timeoutMs: number | null; - maxIterations: number | null; - transportRetries: 0 | null; - reasoningEffort: ModelReasoningEffort; - samplingMode: 'temperature_zero' | 'provider_no_sampling_control' | null; - cacheMode: 'disabled' | null; - pricingProfileId: string | null; - failureDenominatorId: string; - /** Offline validation admits only the explicit non-dispatch terminal state. */ - status: 'not_dispatched'; - finishReason: null; - usage: null; - estimatedCostUsd: null; -} - -/** - * Evaluator-owned pre-dispatch expected values. A coordinator must create and - * authenticate this complete sequence from the immutable manifest before any - * provider call; there is deliberately no implementation in this PR. - */ -export interface FixedTraceTrustedExecutionExpectation { - trustedManifestSha256: string; - planFingerprint: string; - budgetIdentitySha256: string; - entries: readonly FixedTraceRawLedgerEntry[]; -} - -export interface FixedTraceRawAuditableLedger { - version: typeof FIXED_TRACE_RAW_LEDGER_VERSION; - trustedManifestSha256: string; - planFingerprint: string; - /** Exact dry-run reservation identity; summaries cannot substitute it. */ - budgetIdentitySha256: string; - entries: readonly FixedTraceRawLedgerEntry[]; -} -export type FixedTraceRawArtifactResolver = (storageKey: string) => { sha256: string; byteLength: number } | null; - -export interface FixedTraceStageReservation { - armId: string; - repetitionIndex: number; - stage: 'router' | 'generation' | 'judge'; - provider: ModelProviderId; - model: string; - requests: number; - inputBytes: number; - outputTokens: number; - ceilingUsd: number; -} - -export interface FixedTraceDryRunEstimate { - planFingerprint: string; - /** Binds a raw spend ledger to the exact conservative reservations below. */ - budgetIdentitySha256: string; - diagnosticOnly: true; - comparisonEligible: false; - executionOrder: readonly string[]; - candidate: { ceilingUsd: number; expectedSpendUsd: null; reservations: readonly FixedTraceStageReservation[] }; - judges: { ceilingUsd: number; expectedSpendUsd: null; reservations: readonly FixedTraceStageReservation[] }; - totalCeilingUsd: number; - /** No traffic or provider calls occur; expected spend needs observed usage. */ - expectedSpendUsd: null; -} - -export interface FixedTraceOfflinePlanValidation { - diagnosticOnly: true; - comparisonEligible: false; - dispatchable: false; - trustedLock: false; - planFingerprint: string; -} - -function assertExactKeys(value: object, keys: readonly string[], label: string): void { - const actual = Object.keys(value).sort(); - if (actual.some((key) => key === '__proto__' || key === 'prototype' || key === 'constructor')) { - throw new Error(`${label} contains a dangerous prototype key`); - } - const expected = [...keys].sort(); - if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { - throw new Error(`${label} has unknown, missing, or inherited fields`); - } -} - -function canonicalJson(value: unknown): string { - if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); - if (typeof value === 'number') { - if (!Number.isFinite(value)) throw new Error('Cannot fingerprint a non-finite experiment-plan value'); - return JSON.stringify(value); - } - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; - if (typeof value === 'object') { - const record = value as Record; - return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; - } - throw new Error('Cannot fingerprint a non-JSON experiment-plan value'); -} - -function sha256(value: unknown): string { - return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); -} - -function requireHash(value: string, label: string): void { - if (!/^[a-f0-9]{64}$/.test(value)) throw new Error(`${label} must be a SHA-256 hex digest`); -} - -function requirePositiveInteger(value: number, label: string): void { - if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`); -} - -function selectedTraceIds(plan: FixedTraceExperimentPlan): readonly string[] { - return plan.partition.selected === 'development' - ? FIXED_TRACE_PARTITION_MANIFEST.development - : FIXED_TRACE_PARTITION_MANIFEST.repositoryVisibleDevelopmentValidation; -} - -function pricingFor(stage: FixedTracePlannedStage, pricingAsOf: string): FixedTraceImmutablePricingProfile { - const asOf = new Date(pricingAsOf); - if (Number.isNaN(asOf.getTime())) throw new Error('pricingAsOf must be an ISO timestamp'); - const pricing = FIXED_TRACE_IMMUTABLE_PRICING.find((candidate) => - candidate.provider === stage.provider && candidate.model === stage.model && candidate.version === stage.pricingVersion, - ); - if (!pricing) throw new Error(`Unavailable immutable pricing for ${stage.provider}/${stage.model}`); - if (asOf >= new Date(pricing.validBefore)) { - throw new Error(`Stale immutable pricing for ${stage.provider}/${stage.model}`); - } - return pricing; -} - -function validateStage( - stage: FixedTracePlannedStage, - label: string, - traceIds: readonly string[], - pricingAsOf: string, -): FixedTraceImmutablePricingProfile { - assertExactKeys(stage, [ - 'provider', 'model', 'reasoningEffort', 'pricingVersion', 'maxOutputTokens', - 'timeoutMs', 'maxIterations', 'transportRetries', 'samplingMode', 'temperature', - 'cacheMode', 'requestBounds', - ], label); - if (!['anthropic', 'openai', 'google'].includes(stage.provider)) throw new Error(`${label}.provider is unknown`); - if (!['provider_default', 'none', 'low', 'medium', 'high'].includes(stage.reasoningEffort)) throw new Error(`${label}.reasoningEffort is unknown`); - if (!stage.model.trim()) throw new Error(`${label}.model is required`); - requirePositiveInteger(stage.maxOutputTokens, `${label}.maxOutputTokens`); - requirePositiveInteger(stage.timeoutMs, `${label}.timeoutMs`); - requirePositiveInteger(stage.maxIterations, `${label}.maxIterations`); - if ( - (stage.samplingMode === 'temperature_zero' && stage.temperature !== 0) - || (stage.samplingMode === 'provider_no_sampling_control' && stage.temperature !== null) - ) throw new Error(`${label} sampling controls are inconsistent`); - if (stage.transportRetries !== 0 || stage.cacheMode !== 'disabled') { - throw new Error(`${label} has an unsupported retry or cache control`); - } - const pricing = pricingFor(stage, pricingAsOf); - const bounds = stage.requestBounds?.inputBytesByTrace; - if (!bounds || typeof bounds !== 'object') throw new Error(`${label}.requestBounds are required`); - assertExactKeys(stage.requestBounds, ['inputBytesByTrace'], `${label}.requestBounds`); - assertExactKeys(bounds, traceIds, `${label}.requestBounds.inputBytesByTrace`); - for (const traceId of traceIds) { - const values = bounds[traceId]; - if (!Array.isArray(values) || values.length !== stage.maxIterations) { - throw new Error(`${label}.requestBounds must contain ${stage.maxIterations} exact bounds for ${traceId}`); - } - for (const bytes of values) requirePositiveInteger(bytes, `${label}.requestBounds.${traceId}`); - } - if (Object.keys(bounds).some((traceId) => !traceIds.includes(traceId))) { - throw new Error(`${label}.requestBounds contains a trace outside the selected partition`); - } - return pricing; -} - -function validateArm(plan: FixedTraceExperimentPlan, arm: FixedTraceExperimentArm): void { - assertExactKeys(arm, ['id', 'architecture', 'screeningStage', 'repetitionIndex', ...( - arm.router ? ['router'] : []), ...(arm.generation ? ['generation'] : []), ...(arm.judges ? ['judges'] : [])], `arm ${arm.id}`); - if (!['two_stage_llm_router', 'direct_generation', 'hybrid_generation', 'oracle_route_diagnostic'].includes(arm.architecture)) { - throw new Error(`${arm.id}.architecture is unknown`); - } - if (!['router_only_screen', 'oracle_route_generator_diagnostic', 'deployable_finalist'].includes(arm.screeningStage)) { - throw new Error(`${arm.id}.screeningStage is unknown`); - } - if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(arm.id)) throw new Error(`Invalid experiment arm ID: ${arm.id}`); - requirePositiveInteger(arm.repetitionIndex, `${arm.id}.repetitionIndex`); - const traces = selectedTraceIds(plan); - const stages = arm.screeningStage; - if (stages === 'router_only_screen') { - if (arm.architecture !== 'two_stage_llm_router' || !arm.router || arm.generation || (arm.judges?.length ?? 0) !== 0) { - throw new Error(`${arm.id} is not a router-only screening contract`); - } - validateStage(arm.router, `${arm.id}.router`, traces, plan.pricingAsOf); - return; - } - if (stages === 'oracle_route_generator_diagnostic') { - if (arm.architecture !== 'oracle_route_diagnostic' || arm.router || !arm.generation || (arm.judges?.length ?? 0) !== 0) { - throw new Error(`${arm.id} is not an oracle-route diagnostic contract`); - } - validateStage(arm.generation, `${arm.id}.generation`, traces, plan.pricingAsOf); - return; - } - if (arm.architecture !== 'two_stage_llm_router' || !arm.router || !arm.generation) { - throw new Error(`${arm.id} is inadmissible: direct and hybrid execution contracts are not available`); - } - validateStage(arm.router, `${arm.id}.router`, traces, plan.pricingAsOf); - validateStage(arm.generation, `${arm.id}.generation`, traces, plan.pricingAsOf); - if (!arm.judges || arm.judges.length < 2) throw new Error(`${arm.id} requires at least two blinded independent judges`); - const candidateProviders = new Set([arm.router.provider, arm.generation.provider]); - const judgeProviders = new Set(); - for (const [index, judge] of arm.judges.entries()) { - assertExactKeys(judge, [ - 'provider', 'model', 'reasoningEffort', 'pricingVersion', 'maxOutputTokens', - 'timeoutMs', 'maxIterations', 'transportRetries', 'samplingMode', 'temperature', - 'cacheMode', 'requestBounds', 'blinded', - ], `${arm.id}.judges.${index}`); - if (judge.blinded !== true) throw new Error(`${arm.id}.judges.${index} must be blinded`); - if (candidateProviders.has(judge.provider)) throw new Error(`${arm.id}.judges.${index} is not provider-independent`); - judgeProviders.add(judge.provider); - validateStage(judge, `${arm.id}.judges.${index}`, traces, plan.pricingAsOf); - } - if (judgeProviders.size < 2) throw new Error(`${arm.id} requires two provider-independent judges`); -} - -function assertPlanShape(plan: FixedTraceExperimentPlan): void { - assertExactKeys(plan, [ - 'version', 'id', 'trustedManifestId', 'sourceId', 'sourceRevision', 'pricingAsOf', - 'sourceBundleSha256', 'gitCommit', 'gitDirty', 'addieCodeVersion', 'stageControlVersion', - 'traceSuiteSha256', 'promptConfigVersion', 'toolSchemaSha256', 'toolDefinitionProvenance', - 'providerDegradationInjectionEnabled', 'partition', 'ordering', 'budgets', 'arms', - ], 'experiment plan'); - assertExactKeys(plan.partition, ['manifestVersion', 'manifestSha256', 'selected'], 'experiment plan.partition'); - assertExactKeys(plan.ordering, ['seed'], 'experiment plan.ordering'); - assertExactKeys(plan.budgets, ['candidateCeilingUsd', 'judgeCeilingUsd'], 'experiment plan.budgets'); -} - -function assertFixedTraceExperimentPlanStructure( - plan: FixedTraceExperimentPlan, -): void { - assertPlanShape(plan); - assertFixedTracePartitionManifest(); - if (plan.version !== FIXED_TRACE_EXPERIMENT_PLAN_VERSION) throw new Error('Unsupported fixed-trace experiment plan version'); - if (!plan.id.trim()) throw new Error('Experiment plan ID is required'); - if (!plan.trustedManifestId.trim() || !plan.sourceId.trim() || !plan.sourceRevision.trim()) throw new Error('Experiment plan requires a trusted source identity'); - requireHash(plan.sourceBundleSha256, 'sourceBundleSha256'); - if (!/^[a-f0-9]{7,64}$/.test(plan.gitCommit) || typeof plan.gitDirty !== 'boolean' || !plan.addieCodeVersion.trim() || plan.stageControlVersion !== FIXED_TRACE_STAGE_CONTROL_VERSION || typeof plan.providerDegradationInjectionEnabled !== 'boolean') { - throw new Error('Experiment plan run provenance is incomplete'); - } - if (plan.addieCodeVersion !== CODE_VERSION) throw new Error('Experiment plan Addie code version does not match this runner'); - requireHash(plan.traceSuiteSha256, 'traceSuiteSha256'); - requireHash(plan.promptConfigVersion, 'promptConfigVersion'); - requireHash(plan.toolSchemaSha256, 'toolSchemaSha256'); - if (plan.partition.manifestVersion !== FIXED_TRACE_PARTITION_MANIFEST_VERSION || plan.partition.manifestSha256 !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { - throw new Error('Experiment plan uses an uncommitted fixed-trace partition manifest'); - } - if (!['development', 'repository_visible_development_validation'].includes(plan.partition.selected)) { - throw new Error('Only repository-visible development partitions are available'); - } - if (!plan.ordering.seed.trim()) throw new Error('Experiment ordering seed is required'); - if (!Number.isFinite(plan.budgets.candidateCeilingUsd) || plan.budgets.candidateCeilingUsd <= 0) throw new Error('candidateCeilingUsd must be positive'); - if (!Number.isFinite(plan.budgets.judgeCeilingUsd) || plan.budgets.judgeCeilingUsd <= 0) throw new Error('judgeCeilingUsd must be positive'); - if (!Array.isArray(plan.arms) || plan.arms.length === 0) throw new Error('Experiment plan requires at least one arm'); - const ids = new Set(); - for (const arm of plan.arms) { - if (ids.has(arm.id)) throw new Error(`Duplicate experiment arm ID: ${arm.id}`); - ids.add(arm.id); - validateArm(plan, arm); - } -} - -/** The sole boundary at which caller data becomes immutable plan data. */ -function validatedPlanSnapshot(plan: FixedTraceExperimentPlan): FixedTraceExperimentPlan { - const snapshot = snapshotFixedTraceJson(plan, 'experiment plan') as FixedTraceExperimentPlan; - assertFixedTraceExperimentPlanStructure(snapshot); - return snapshot; -} - -/** Validate an untrusted plan without credentials, providers, outputs, or a resolver. */ -export function validateFixedTraceExperimentPlanOffline(plan: FixedTraceExperimentPlan): FixedTraceOfflinePlanValidation { - const snapshot = validatedPlanSnapshot(plan); - // A submitted plan can describe only priced, already reviewed stages. Terra - // and Sol have no reviewed repository price, so their descriptors cannot - // enter an estimate or a budget reservation. - return Object.freeze({ - diagnosticOnly: true, - comparisonEligible: false, - dispatchable: false, - trustedLock: false, - planFingerprint: sha256(snapshot), - }); -} - -/** - * Validates an offline ledger in its exact planned order. Nothing in this - * lane can have been dispatched, returned, priced, or promoted; accepting a - * partial or provider-shaped record would let caller data masquerade as - * evidence. - */ -export function validateFixedTraceRawAuditableLedgerOffline( - plan: FixedTraceExperimentPlan, - ledger: FixedTraceRawAuditableLedger, - expectedTrustedManifestSha256: string, -): void { - // Do not bless syntax-shaped caller evidence. Exact expected prompts, - // tool surface/order, request/envelope/admission hashes, simulator - // provenance, per-invocation attempts, returned identity, usage, pricing, - // cost, and failure denominator must be bound by a trusted coordinator - // before dispatch. That coordinator is deliberately not authorized here. - void plan; - void ledger; - void expectedTrustedManifestSha256; - throw new Error('Raw-ledger validation is unavailable pending a trusted evaluator-owned coordinator'); -} - -/** - * There is deliberately no resolver in this change that can turn caller JSON - * into evaluator authority. A future reviewed evaluator must authenticate a - * manifest outside this process before exposing an execution binding. - */ -function resolveTrustedManifest( - _plan: FixedTraceExperimentPlan, - _resolver: FixedTraceTrustedManifestResolver, -): FixedTraceTrustedManifest { - throw new Error('Trusted fixed-trace manifest is locked pending evaluator-owned authentication'); -} - -/** Builds an immutable input binding for exactly one planned arm, never a dispatcher. */ -export function fixedTraceExperimentRunnerBinding( - plan: FixedTraceExperimentPlan, - resolver: FixedTraceTrustedManifestResolver, - armId: string, -): FixedTraceExperimentRunnerBinding { - assertFixedTraceExperimentPlan(plan, resolver); - const arm = plan.arms.find((candidate) => candidate.id === armId); - if (!arm) throw new Error(`Experiment plan has no arm: ${armId}`); - const manifest = resolveTrustedManifest(plan, resolver); - const suite = manifest.suites[plan.partition.selected]; - return Object.freeze({ - runId: `${plan.id}:${arm.id}:r${arm.repetitionIndex}`, - repetition: arm.repetitionIndex, - sourceBundleSha256: plan.sourceBundleSha256, - gitCommit: plan.gitCommit, - gitDirty: plan.gitDirty, - addieCodeVersion: plan.addieCodeVersion, - stageControlVersion: plan.stageControlVersion, - promptConfigVersion: plan.promptConfigVersion, - traceSuite: deepFreezeFixedTrace(snapshotFixedTraceJson(suite.traceSuite, 'trusted manifest trace suite')) as ReadonlyArray, - traceSuiteSha256: suite.traceSuiteSha256, - toolDefinitions: deepFreezeFixedTrace(snapshotFixedTraceJson(suite.toolDefinitions, 'trusted manifest tool definitions')) as ReadonlyArray, - toolDefinitionProvenance: suite.toolDefinitionProvenance, - providerDegradationInjectionEnabled: plan.providerDegradationInjectionEnabled, - }); -} - -/** Omits partition selection so a candidate description cannot be restamped as a different development split. */ -export function fixedTraceCandidatePlanFingerprint(plan: FixedTraceExperimentPlan): string { - const snapshot = validatedPlanSnapshot(plan); - const { partition, ...candidatePlan } = snapshot; - return sha256({ - ...candidatePlan, - partition: { - manifestVersion: partition.manifestVersion, - manifestSha256: partition.manifestSha256, - }, - }); -} - -export function assertFixedTraceExperimentPlan( - plan: FixedTraceExperimentPlan, - resolver: FixedTraceTrustedManifestResolver, -): void { - const snapshot = validatedPlanSnapshot(plan); - resolveTrustedManifest(snapshot, resolver); -} - -export function fixedTraceExperimentPlanFingerprint(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): string { - void resolver; - return validateFixedTraceExperimentPlanOffline(plan).planFingerprint; -} - -/** Deterministic permutation based on a recorded seed, never provider input order. */ -export function fixedTraceExperimentExecutionOrder(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): readonly string[] { - void resolver; - const snapshot = validatedPlanSnapshot(plan); - return Object.freeze([...snapshot.arms] - .sort((left, right) => sha256({ seed: snapshot.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) - .localeCompare(sha256({ seed: snapshot.ordering.seed, arm: right.id, repetition: right.repetitionIndex })) || left.id.localeCompare(right.id)) - .map((arm) => arm.id)); -} - -function reservation( - arm: FixedTraceExperimentArm, - stageName: FixedTraceStageReservation['stage'], - stage: FixedTracePlannedStage, - traceIds: readonly string[], - pricingAsOf: string, -): FixedTraceStageReservation { - const pricing = pricingFor(stage, pricingAsOf); - const inputBytes = traceIds.reduce((total, traceId) => total + stage.requestBounds.inputBytesByTrace[traceId].reduce((sum, bytes) => sum + bytes, 0), 0); - const requests = traceIds.length * stage.maxIterations; - const outputTokens = requests * stage.maxOutputTokens; - const ceilingUsd = ( - inputBytes * pricing.inputUsdPerMillionTokens + outputTokens * pricing.outputUsdPerMillionTokens - ) / 1_000_000; - return Object.freeze({ armId: arm.id, repetitionIndex: arm.repetitionIndex, stage: stageName, provider: stage.provider, model: stage.model, requests, inputBytes, outputTokens, ceilingUsd }); -} - -/** - * Pure pre-dispatch ceiling. It reports no expected spend because neither - * provider tokenization nor observed tool-loop length may be assumed. - */ -export function estimateFixedTraceExperiment(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): FixedTraceDryRunEstimate { - void resolver; - const snapshot = validatedPlanSnapshot(plan); - const candidate: FixedTraceStageReservation[] = []; - const judges: FixedTraceStageReservation[] = []; - const traceIds = selectedTraceIds(snapshot); - for (const arm of snapshot.arms) { - if (arm.router) candidate.push(reservation(arm, 'router', arm.router, traceIds, snapshot.pricingAsOf)); - if (arm.generation) candidate.push(reservation(arm, 'generation', arm.generation, traceIds, snapshot.pricingAsOf)); - for (const judge of arm.judges ?? []) judges.push(reservation(arm, 'judge', judge, traceIds, snapshot.pricingAsOf)); - } - const candidateCeilingUsd = candidate.reduce((total, item) => total + item.ceilingUsd, 0); - const judgeCeilingUsd = judges.reduce((total, item) => total + item.ceilingUsd, 0); - if (candidateCeilingUsd > snapshot.budgets.candidateCeilingUsd) throw new Error('Candidate worst-case reservation exceeds its separate budget'); - if (judgeCeilingUsd > snapshot.budgets.judgeCeilingUsd) throw new Error('Judge worst-case reservation exceeds its separate budget'); - const planFingerprint = sha256(snapshot); - const budgetIdentitySha256 = sha256({ - planFingerprint, - candidate: candidate.map((item) => ({ ...item })), - judges: judges.map((item) => ({ ...item })), - }); - return Object.freeze({ - planFingerprint, - budgetIdentitySha256, - diagnosticOnly: true, - comparisonEligible: false, - executionOrder: Object.freeze([...snapshot.arms] - .sort((left, right) => sha256({ seed: snapshot.ordering.seed, arm: left.id, repetition: left.repetitionIndex }) - .localeCompare(sha256({ seed: snapshot.ordering.seed, arm: right.id, repetition: right.repetitionIndex })) || left.id.localeCompare(right.id)) - .map((arm) => arm.id)), - candidate: Object.freeze({ ceilingUsd: candidateCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(candidate) }), - judges: Object.freeze({ ceilingUsd: judgeCeilingUsd, expectedSpendUsd: null, reservations: Object.freeze(judges) }), - totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd, - expectedSpendUsd: null, - }); -} - -/** This is an ID-only development-validation audit, never a secret holdout. */ -export function fixedTraceExperimentPartitionAudit(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): { selected: 'development' | 'repository_visible_development_validation'; traceIds: readonly string[]; manifestSha256: string; limitation: typeof FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION } { - assertFixedTraceExperimentPlan(plan, resolver); - return Object.freeze({ selected: plan.partition.selected, traceIds: Object.freeze([...selectedTraceIds(plan)]), manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, limitation: FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION }); -} - -/** Development selection artifacts cannot claim any confirmatory final metrics. */ -export function fixedTraceDevelopmentSelectionArtifact(plan: FixedTraceExperimentPlan, resolver: FixedTraceTrustedManifestResolver): { planFingerprint: string; developmentTraceIds: readonly string[]; confirmatoryMetricsIncluded: false; limitation: typeof FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION } { - assertFixedTraceExperimentPlan(plan, resolver); - if (plan.partition.selected !== 'development') throw new Error('Repository-visible validation cannot be emitted as a development selection artifact'); - return Object.freeze({ planFingerprint: fixedTraceExperimentPlanFingerprint(plan, resolver), developmentTraceIds: Object.freeze([...FIXED_TRACE_PARTITION_MANIFEST.development]), confirmatoryMetricsIncluded: false, limitation: FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION }); -} - -/** - * Validates raw-auditable execution provenance before anything can be used in - * comparison or rollout. This does not score or promote a candidate: the - * repaired foundation owns evidence verification and must consume this ledger. - */ -export function assertFixedTraceRawAuditableLedger( - plan: FixedTraceExperimentPlan, - resolver: FixedTraceTrustedManifestResolver, - ledger: FixedTraceRawAuditableLedger, - artifactResolver: FixedTraceRawArtifactResolver, -): void { - // No caller-provided ledger can be evidence until the missing coordinator - // constructs and authenticates exact expected invocation records. - void plan; - void resolver; - void ledger; - void artifactResolver; - throw new Error('Raw-ledger validation is unavailable pending a trusted evaluator-owned coordinator'); -} diff --git a/server/src/addie/eval/fixed-trace-partition.ts b/server/src/addie/eval/fixed-trace-partition.ts index 97fa32f89e..e98f7a8b4a 100644 --- a/server/src/addie/eval/fixed-trace-partition.ts +++ b/server/src/addie/eval/fixed-trace-partition.ts @@ -1,57 +1,76 @@ -import { createHash } from 'node:crypto'; +import { createHash } from "node:crypto"; +import { + FIXED_TRACE_CORPUS, + FIXED_TRACE_PHASE_COUNTS, +} from "./fixed-trace-suite.js"; /** * This ID-only manifest is the partition boundary. It deliberately contains * no fixture text, expected routes, or grading rubric. */ -export const FIXED_TRACE_PARTITION_MANIFEST_VERSION = 'addie-fixed-trace-partition-v1' as const; +export const FIXED_TRACE_PARTITION_MANIFEST_VERSION = + "addie-fixed-trace-partition-v2" as const; export const FIXED_TRACE_PARTITION_MANIFEST = Object.freeze({ version: FIXED_TRACE_PARTITION_MANIFEST_VERSION, - development: Object.freeze([ - 'surface-channel-chatter', 'knowledge-task-model', 'community-discussion-search-read-only', - 'member-own-profile', 'member-company-listing', 'sponsored-intelligence-agent-discovery', - 'sponsored-intelligence-session-status', 'committee-co-leader-read-only', 'publishing-own-submissions', - 'publishing-cover-status', 'brand-mutual-assertion', 'adcp-saved-agent-list', 'directory-agent-lookup', - 'property-identifier-catalog-browse', 'admin-duplicate-organizations', 'admin-member-records-without-slack', - 'admin-brand-logo-review', 'admin-billing-pending-invoices', 'admin-prospect-pipeline-query', - 'admin-feed-monitoring-proposals', 'admin-followup-task-list', 'outreach-action-items-list', - 'meeting-full-administration-confirmed', 'community-group-full-participation-confirmed', - ]), - // This is intentionally *not* a holdout. Every request, route, - // expectation, rubric, and fixture is repository-visible, so it is usable - // only for development validation. A confirmatory pack is absent from this - // repository and must be externally authored and custodied. - repositoryVisibleDevelopmentValidation: Object.freeze([ - 'billing-invoice-preview-only', 'billing-invoice-confirmed', 'knowledge-tool-error', - 'tool-result-prompt-injection', 'current-utc-date', 'bounded-truncation', - 'long-form-deck-delivery', 'provider-unavailable', - ]), + /** Derived from the corpus authority, never a stale hand-copied list. */ + development: Object.freeze( + FIXED_TRACE_CORPUS.filter((trace) => trace.phase === "development").map( + (trace) => trace.id, + ), + ), + tuning: Object.freeze( + FIXED_TRACE_CORPUS.filter((trace) => trace.phase === "tuning").map( + (trace) => trace.id, + ), + ), }); -export const FIXED_TRACE_PARTITION_MANIFEST_SHA256 = - '65407c60fce215042c1e692ac2edbc7f501d7b83802661e1d3dd61fafde4b74b' as const; - function canonicalJson(value: unknown): string { - if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; - if (typeof value === 'object') { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { const record = value as Record; - return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; } - throw new Error('Partition manifest contains a non-JSON value'); + throw new Error("Partition manifest contains a non-JSON value"); } export function fixedTracePartitionManifestSha256(): string { - return createHash('sha256').update(canonicalJson(FIXED_TRACE_PARTITION_MANIFEST), 'utf8').digest('hex'); + return createHash("sha256") + .update(canonicalJson(FIXED_TRACE_PARTITION_MANIFEST), "utf8") + .digest("hex"); } +export const FIXED_TRACE_PARTITION_MANIFEST_SHA256 = + fixedTracePartitionManifestSha256(); + export function assertFixedTracePartitionManifest(): void { - if (fixedTracePartitionManifestSha256() !== FIXED_TRACE_PARTITION_MANIFEST_SHA256) { - throw new Error('Fixed-trace partition manifest hash mismatch'); + if ( + fixedTracePartitionManifestSha256() !== + FIXED_TRACE_PARTITION_MANIFEST_SHA256 + ) { + throw new Error("Fixed-trace partition manifest hash mismatch"); } const all = [ ...FIXED_TRACE_PARTITION_MANIFEST.development, - ...FIXED_TRACE_PARTITION_MANIFEST.repositoryVisibleDevelopmentValidation, + ...FIXED_TRACE_PARTITION_MANIFEST.tuning, ]; - if (new Set(all).size !== all.length) throw new Error('Fixed-trace partition manifest has duplicate IDs'); + if (new Set(all).size !== all.length) + throw new Error("Fixed-trace partition manifest has duplicate IDs"); + if ( + FIXED_TRACE_PARTITION_MANIFEST.development.length !== + FIXED_TRACE_PHASE_COUNTS.development || + FIXED_TRACE_PARTITION_MANIFEST.tuning.length !== + FIXED_TRACE_PHASE_COUNTS.tuning || + FIXED_TRACE_PHASE_COUNTS.sealed_final !== 0 || + all.length !== 82 + ) { + throw new Error( + "Fixed-trace partition manifest does not match the 46 development / 36 tuning corpus authority", + ); + } } diff --git a/server/src/addie/model-cost-pricing.ts b/server/src/addie/model-cost-pricing.ts index 3d2c8c6720..75b1b031ef 100644 --- a/server/src/addie/model-cost-pricing.ts +++ b/server/src/addie/model-cost-pricing.ts @@ -10,16 +10,19 @@ import { CLAUDE_PRICING_VERSION, costUsdMicros, resolveKnownClaudePricingModel, -} from './claude-pricing.js'; +} from "./claude-pricing.js"; import { GOOGLE_ROUTER_MODEL, isGoogleRouterModelRevision, -} from './model-providers/google-generate-content-provider.js'; -import { OPENAI_ROUTER_MODEL } from './model-providers/openai-responses-provider.js'; -import type { ModelProviderId, ModelUsage } from './model-providers/model-provider.js'; +} from "./model-providers/google-generate-content-provider.js"; +import { OPENAI_ROUTER_MODEL } from "./model-providers/openai-responses-provider.js"; +import type { + ModelProviderId, + ModelUsage, +} from "./model-providers/model-provider.js"; export const GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION = - 'google-gemini-3.7-flash-through-2026-12-31' as const; + "google-gemini-3.7-flash-through-2026-12-31" as const; /** * This is the existing, reviewed Luna router price identity used by the @@ -27,7 +30,25 @@ export const GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION = * reviewed price entry and must not inherit Luna's availability or rate. */ export const OPENAI_GPT_5_6_LUNA_PRICING_VERSION = - 'openai-gpt-5.6-luna-2026-08-26' as const; + "openai-gpt-5.6-luna-2026-08-26" as const; + +/** + * The single reviewed Luna price identity. Fixed-trace planning, reservation, + * and settlement import this record rather than copying a near-match profile. + * `inputTokens` includes cached input, so cached input is a subset replacement + * (not an additional charge and not an uncached charge). + */ +export const OPENAI_GPT_5_6_LUNA_PRICING = Object.freeze({ + profileId: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, + inputUsdPerMillionTokens: 0.2, + outputUsdPerMillionTokens: 1.2, + cacheReadUsdPerMillionTokens: 0.02, + cacheReadAccounting: "subset" as const, + cacheWriteUsdPerMillionTokens: null, + cacheWriteAccounting: "unsupported" as const, + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", +}); export interface ModelCostPricing { provider: ModelProviderId; @@ -39,14 +60,18 @@ export interface ModelCostPricing { /** A complete provider-normalized usage tuple is required for live charging. */ export function hasCompleteModelUsage(usage: unknown): usage is ModelUsage { - if (!usage || typeof usage !== 'object') return false; + if (!usage || typeof usage !== "object") return false; const value = usage as Record; const isSafeCount = (count: unknown): count is number => - typeof count === 'number' && Number.isSafeInteger(count) && count >= 0; - return isSafeCount(value.inputTokens) - && isSafeCount(value.outputTokens) - && (value.cacheReadTokens === undefined || isSafeCount(value.cacheReadTokens)) - && (value.cacheWriteTokens === undefined || isSafeCount(value.cacheWriteTokens)); + typeof count === "number" && Number.isSafeInteger(count) && count >= 0; + return ( + isSafeCount(value.inputTokens) && + isSafeCount(value.outputTokens) && + (value.cacheReadTokens === undefined || + isSafeCount(value.cacheReadTokens)) && + (value.cacheWriteTokens === undefined || + isSafeCount(value.cacheWriteTokens)) + ); } /** @@ -59,50 +84,60 @@ export function resolveModelCostPricing( provider: ModelProviderId | string, model: string, ): ModelCostPricing | null { - if (provider === 'openai' && model === OPENAI_ROUTER_MODEL) { + if (provider === "openai" && model === OPENAI_ROUTER_MODEL) { return { - provider: 'openai', + provider: "openai", model: OPENAI_ROUTER_MODEL, version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, validBefore: null, - // Official standard pricing checked 2026-08-26: - // https://developers.openai.com/api/docs/models/gpt-5.6-luna - estimateCostMicros: (usage) => Math.ceil( - usage.inputTokens * 0.2 + usage.outputTokens * 1.2, - ), + estimateCostMicros: (usage) => { + const cacheReadTokens = usage.cacheReadTokens ?? 0; + const uncachedInputTokens = + cacheReadTokens <= usage.inputTokens + ? usage.inputTokens - cacheReadTokens + : usage.inputTokens; + return Math.ceil( + uncachedInputTokens * + OPENAI_GPT_5_6_LUNA_PRICING.inputUsdPerMillionTokens + + cacheReadTokens * + OPENAI_GPT_5_6_LUNA_PRICING.cacheReadUsdPerMillionTokens + + usage.outputTokens * + OPENAI_GPT_5_6_LUNA_PRICING.outputUsdPerMillionTokens, + ); + }, }; } - const canonicalAnthropicModel = provider === 'anthropic' - ? resolveKnownClaudePricingModel(model) - : null; - if (provider === 'anthropic' && canonicalAnthropicModel) { + const canonicalAnthropicModel = + provider === "anthropic" ? resolveKnownClaudePricingModel(model) : null; + if (provider === "anthropic" && canonicalAnthropicModel) { return { - provider: 'anthropic', + provider: "anthropic", model, version: `${CLAUDE_PRICING_VERSION}:${canonicalAnthropicModel}`, validBefore: null, - estimateCostMicros: (usage) => costUsdMicros(canonicalAnthropicModel, { - input_tokens: usage.inputTokens, - output_tokens: usage.outputTokens, - cache_read_input_tokens: usage.cacheReadTokens, - cache_creation_input_tokens: usage.cacheWriteTokens, - }), + estimateCostMicros: (usage) => + costUsdMicros(canonicalAnthropicModel, { + input_tokens: usage.inputTokens, + output_tokens: usage.outputTokens, + cache_read_input_tokens: usage.cacheReadTokens, + cache_creation_input_tokens: usage.cacheWriteTokens, + }), }; } // Google Generate Content accepts this canonical router model and its // provider-returned eight-digit dated revisions (for example `...-20260801`). Keep this // mapping here, beside the reviewed rate, rather than falling back to any // other model or provider price. - const canonicalGoogleModel = provider === 'google' - && isGoogleRouterModelRevision(model) - ? GOOGLE_ROUTER_MODEL - : null; + const canonicalGoogleModel = + provider === "google" && isGoogleRouterModelRevision(model) + ? GOOGLE_ROUTER_MODEL + : null; if (canonicalGoogleModel) { return { - provider: 'google', + provider: "google", model, version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - validBefore: new Date('2027-01-01T00:00:00.000Z'), + validBefore: new Date("2027-01-01T00:00:00.000Z"), // Official standard pricing checked 2026-08-30: $0.75/M input, // $0.075/M cached input, and $3.75/M output (including thought tokens). // A cache-read count above input is charged in addition to all input, @@ -110,14 +145,15 @@ export function resolveModelCostPricing( estimateCostMicros: (usage) => { const cacheReadTokens = usage.cacheReadTokens ?? 0; const cacheWriteTokens = usage.cacheWriteTokens ?? 0; - const uncachedInput = cacheReadTokens <= usage.inputTokens - ? usage.inputTokens - cacheReadTokens - : usage.inputTokens; + const uncachedInput = + cacheReadTokens <= usage.inputTokens + ? usage.inputTokens - cacheReadTokens + : usage.inputTokens; return Math.ceil( - uncachedInput * 0.75 - + cacheReadTokens * 0.075 - + cacheWriteTokens * 0.75 - + usage.outputTokens * 3.75, + uncachedInput * 0.75 + + cacheReadTokens * 0.075 + + cacheWriteTokens * 0.75 + + usage.outputTokens * 3.75, ); }, }; diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index d586fcf78c..f77e260424 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -1,381 +1,31 @@ -/** - * Fixed-trace experiment-plan dry run. - * - * Production handlers and production messages are never loaded into the - * executor: every tool result comes from the immutable fixed-trace fixtures. - * The required shared soft budget admits each exact prepared request before - * dispatch and halts after unknown spend exposure. - * - * `--architecture-arm=direct_generation` is intentionally admission-only. - * Production builds an authorization-aware definition/handler intersection - * before intent narrowing, but this harness neither captures that intersection - * nor bounds it independently; fixture-local schemas must not stand in for it. - * `oracle_route_diagnostic` may execute generation with fixture routing. - * The hybrid-only `--suite=hybrid-evaluator` binds the separately reviewed - * local-admission corpus without altering the legacy 32 traces. Every arm is - * diagnostic-only in this foundation: independent judging, - * comparison, and rollout are blocked until an evaluator-owned run-context - * and raw-ledger coordinator can authenticate serialized artifacts. - * - * This legacy entrypoint is intentionally planning-only while the execution - * adapter is being separated from the production path. It never constructs a - * provider or reads credentials. A live replay must be added as a separately - * reviewed consumer of the versioned plan contract. - * - * `--experiment-plan` and `--trusted-manifest` are accepted only with - * `--validate-only`; neither can make a caller-built manifest trusted. - */ -import { createHash, randomUUID } from 'node:crypto'; -import { execFileSync } from 'node:child_process'; -import { resolve } from 'node:path'; -import { readFileSync } from 'node:fs'; -import { ModelConfig } from '../../src/config/models.js'; -import { CODE_VERSION, computeRouterRulesHash } from '../../src/addie/config-version.js'; +/** Planning-only manual entrypoint: it has no dispatch or output path. */ +import { parseFixedTraceDiagnosticCliArguments } from "../../src/addie/eval/fixed-trace-diagnostic-cli.js"; import { - BudgetedFixedTraceProvider, - FixedTraceBudget, - fixedTraceResponsePricingPolicy, -} from '../../src/addie/eval/fixed-trace-budget.js'; -import { - type FixedTraceProviderStageConfig, -} from '../../src/addie/eval/fixed-trace-runner.js'; -import { - runFixedTraceDiagnosticArtifact, - type FixedTraceDiagnosticProviderPlan, -} from '../../src/addie/eval/fixed-trace-diagnostic-run.js'; -import { MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS } from '../../src/addie/eval/fixed-trace-tool-loop.js'; -import { parseFixedTraceDiagnosticCliArguments } from '../../src/addie/eval/fixed-trace-diagnostic-cli.js'; -import { validateFixedTraceExperimentPlanOffline } from '../../src/addie/eval/fixed-trace-experiment-plan.js'; -import { reserveFixedTraceDiagnosticOutput } from '../../src/addie/eval/fixed-trace-diagnostic-output.js'; -import { canonicalFixedTraceToolDefinitions } from '../../src/addie/eval/fixed-trace-tools.js'; -import { - fixedTraceHybridPolicy, - type FixedTraceArchitectureArmId, -} from '../../src/addie/eval/fixed-trace-architecture.js'; -import { - FIXED_TRACE_SUITE, - FIXED_TRACE_HYBRID_EVALUATOR_SUITE, - fixedTraceSuiteSha256, - type FixedTracePricing, -} from '../../src/addie/eval/fixed-trace-suite.js'; -import { AnthropicRouterProvider } from '../../src/addie/model-providers/anthropic-router-provider.js'; -import { AnthropicModelProvider } from '../../src/addie/model-providers/anthropic-provider.js'; -import type { - ModelProvider, - ModelProviderId, - ModelReasoningEffort, -} from '../../src/addie/model-providers/model-provider.js'; -import { - OpenAIResponsesProvider, - OPENAI_ROUTER_MODEL, -} from '../../src/addie/model-providers/openai-responses-provider.js'; -import { - GoogleGenerateContentProvider, - GOOGLE_ROUTER_MODEL, -} from '../../src/addie/model-providers/google-generate-content-provider.js'; -import { loadResponseStyle, loadRules } from '../../src/addie/rules/index.js'; - -type ProviderName = ModelProviderId; - -type ProviderPlan = FixedTraceDiagnosticProviderPlan & { name: ProviderName }; - -const PRICING = { - anthropicRouter: { - profileId: 'anthropic-standard-2026-08:claude-haiku-4-5', - inputUsdPerMillionTokens: 1, - outputUsdPerMillionTokens: 5, - cacheReadUsdPerMillionTokens: 0.1, - cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.', - }, - anthropicGeneration: { - profileId: 'anthropic-standard-2026-08:claude-sonnet-5', - inputUsdPerMillionTokens: 3, - outputUsdPerMillionTokens: 15, - cacheReadUsdPerMillionTokens: 0.3, - cacheWriteUsdPerMillionTokens: 3.75, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Sonnet 5 standard, refreshed August 2026.', - }, - openai: { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', - inputUsdPerMillionTokens: 0.2, - outputUsdPerMillionTokens: 1.2, - cacheReadUsdPerMillionTokens: 0.02, - cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'unsupported', - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', - }, - google: { - profileId: 'google-gemini-3.7-flash-through-2026-12-31', - inputUsdPerMillionTokens: 0.75, - outputUsdPerMillionTokens: 3.75, - cacheReadUsdPerMillionTokens: 0.075, - cacheWriteUsdPerMillionTokens: 0.75, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'additive', - source: 'Google Gemini 3.7 Flash introductory standard, checked 2026-08-25.', - }, -} satisfies Record; - -const cliArguments = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); - -function argument(name: string): string | undefined { - return cliArguments[{ providers: 'providers', 'architecture-arm': 'architectureArm', suite: 'suite', 'soft-max-usd': 'softMaxUsd', output: 'output', 'experiment-plan': 'experimentPlan', 'trusted-manifest': 'trustedManifest' }[name] as keyof typeof cliArguments] as string | undefined; -} - -function sha256(value: string): string { - return createHash('sha256').update(value, 'utf8').digest('hex'); -} - -function sourceBundle(): { sha256: string; files: string[] } { - const trackedFiles = execFileSync('git', [ - 'ls-files', '-z', 'package.json', 'package-lock.json', 'server/src/addie', - 'server/src/config/models.ts', 'server/tests/manual/fixed-trace-provider-eval.ts', - ], { encoding: 'utf8' }).split('\0').filter(Boolean).sort(); - const files = [...new Set([ - ...trackedFiles, - 'server/src/addie/eval/fixed-trace-budget.ts', - 'server/src/addie/eval/fixed-trace-architecture.ts', - 'server/src/addie/eval/fixed-trace-runner.ts', - 'server/tests/manual/fixed-trace-provider-eval.ts', - ])].sort(); - const hash = createHash('sha256'); - for (const file of files) { - hash.update(file, 'utf8').update('\0').update(readFileSync(file)).update('\0'); - } - return { sha256: hash.digest('hex'), files }; -} - -function stage( - provider: ModelProvider, - model: string, - reasoningEffort: ModelReasoningEffort, - maxOutputTokens: number, - maxIterations: number, - pricing: FixedTracePricing, -): FixedTraceProviderStageConfig { - return { - provider, - model, - reasoningEffort, - maxOutputTokens, - timeoutMs: 120_000, - maxIterations, - transportRetries: 0, - samplingMode: 'provider_no_sampling_control', - temperature: null, - pricing, - }; -} - -function budgetedStageProvider( - provider: ModelProvider, - budget: FixedTraceBudget, - model: string, - pricing: FixedTracePricing, -): BudgetedFixedTraceProvider { - return new BudgetedFixedTraceProvider( - provider, - budget, - pricing, - fixedTraceResponsePricingPolicy(provider.id, model, pricing), + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + assertFixedTraceEvaluationProtocol, + estimateFixedTraceEvaluationProtocol, +} from "../../src/addie/eval/fixed-trace-evaluation-protocol.js"; + +const arguments_ = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); +if (!arguments_.validateOnly) + throw new Error( + "This planning-only evaluator requires --validate-only and cannot dispatch providers", ); -} - -function providerPlans( - names: readonly ProviderName[], - budget: FixedTraceBudget, -): ProviderPlan[] { - const plans: ProviderPlan[] = []; - if (names.includes('anthropic')) { - if (!process.env.ANTHROPIC_API_KEY) throw new Error('ANTHROPIC_API_KEY is required'); - if (ModelConfig.fast !== 'claude-haiku-4-5') throw new Error('Fixed traces pin Anthropic routing to claude-haiku-4-5'); - if (ModelConfig.primary !== 'claude-sonnet-5') throw new Error('Fixed traces pin Anthropic generation to claude-sonnet-5'); - const router = new AnthropicRouterProvider(process.env.ANTHROPIC_API_KEY, { maxRetries: 0 }); - const generation = new AnthropicModelProvider( - process.env.ANTHROPIC_API_KEY, - undefined, - { transportMaxRetries: 0 }, - ); - const budgetedGeneration = budgetedStageProvider(generation, budget, ModelConfig.primary, PRICING.anthropicGeneration); - plans.push({ - name: 'anthropic', - router: stage( - budgetedStageProvider(router, budget, ModelConfig.fast, PRICING.anthropicRouter), - ModelConfig.fast, - 'provider_default', - 300, - 1, - PRICING.anthropicRouter, - ), - generation: stage( - budgetedGeneration, - ModelConfig.primary, - 'provider_default', - 900, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.anthropicGeneration, - ), - }); - } - if (names.includes('openai')) { - if (!process.env.OPENAI_API_KEY) throw new Error('OPENAI_API_KEY is required'); - const provider = new OpenAIResponsesProvider(process.env.OPENAI_API_KEY); - const budgetedProvider = budgetedStageProvider(provider, budget, OPENAI_ROUTER_MODEL, PRICING.openai); - plans.push({ - name: 'openai', - router: stage( - budgetedProvider, - OPENAI_ROUTER_MODEL, - 'none', - 300, - 1, - PRICING.openai, - ), - generation: stage( - budgetedProvider, - OPENAI_ROUTER_MODEL, - 'none', - 900, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.openai, - ), - }); - } - if (names.includes('google')) { - if (!process.env.GEMINI_API_KEY) throw new Error('GEMINI_API_KEY is required'); - const provider = new GoogleGenerateContentProvider(process.env.GEMINI_API_KEY); - const budgetedProvider = budgetedStageProvider(provider, budget, GOOGLE_ROUTER_MODEL, PRICING.google); - plans.push({ - name: 'google', - router: stage( - budgetedProvider, - GOOGLE_ROUTER_MODEL, - 'low', - 1_200, - 1, - PRICING.google, - ), - generation: stage( - budgetedProvider, - GOOGLE_ROUTER_MODEL, - 'low', - 1_200, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.google, - ), - }); - } - return plans; -} - -const providerNames = (argument('providers') ?? 'anthropic,openai,google').split(',') as ProviderName[]; -if (providerNames.some((name) => !['anthropic', 'openai', 'google'].includes(name))) { - throw new Error('Unknown --providers value'); -} -if (new Set(providerNames).size !== providerNames.length || providerNames.length === 0) { - throw new Error('--providers must contain one or more unique providers'); -} -const architectureArm = (argument('architecture-arm') ?? 'two_stage_llm_router') as FixedTraceArchitectureArmId; -if (!(architectureArm in { two_stage_llm_router: true, direct_generation: true, deterministic_policy_llm_fallback_hybrid: true, oracle_route_diagnostic: true })) { - throw new Error('Unknown --architecture-arm value'); -} -const suiteName = argument('suite') ?? 'canonical'; -if (suiteName !== 'canonical' && suiteName !== 'hybrid-evaluator') throw new Error('Unknown --suite value'); -if (suiteName === 'hybrid-evaluator' && architectureArm !== 'deterministic_policy_llm_fallback_hybrid') { - throw new Error('--suite=hybrid-evaluator requires --architecture-arm=deterministic_policy_llm_fallback_hybrid'); -} -const traceSuite = suiteName === 'hybrid-evaluator' - ? FIXED_TRACE_HYBRID_EVALUATOR_SUITE - : FIXED_TRACE_SUITE; -const softMaxUsd = Number(argument('soft-max-usd')); -if (!Number.isFinite(softMaxUsd) || softMaxUsd <= 0) { - throw new Error('--soft-max-usd is required and must be positive'); -} -const outputArgument = argument('output'); -if (cliArguments.validateOnly) { - const experimentPlanPath = argument('experiment-plan'); - const trustedManifestPath = argument('trusted-manifest'); - if ((experimentPlanPath === undefined) !== (trustedManifestPath === undefined)) { - throw new Error('--experiment-plan and --trusted-manifest must be supplied together'); - } - const offlinePlan = experimentPlanPath - ? validateFixedTraceExperimentPlanOffline(JSON.parse(readFileSync(resolve(experimentPlanPath), 'utf8'))) - : undefined; - // Deliberately parse for malformed-file feedback but never deserialize this - // into a resolver or treat it as authority. Authentication is absent here. - if (trustedManifestPath) JSON.parse(readFileSync(resolve(trustedManifestPath), 'utf8')); - console.log(JSON.stringify({ +if (arguments_.output !== undefined) + throw new Error( + "--output is unavailable in validate-only mode; no artifact may be written", + ); +assertFixedTraceEvaluationProtocol(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); +const estimate = estimateFixedTraceEvaluationProtocol( + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, +); +console.log( + JSON.stringify({ diagnosticOnly: true, - judgeDispatch: 'blocked_pending_trusted_evaluator_owned_coordinator', - validated: { - providers: providerNames, - architectureArm, - suite: suiteName, - softMaxUsd, - outputPath: outputArgument ? resolve(outputArgument) : undefined, - trustedManifestPath: trustedManifestPath ? resolve(trustedManifestPath) : undefined, - offlinePlan, - }, - })); - process.exit(0); -} -if (!outputArgument?.trim()) throw new Error('--output is required'); -resolve(outputArgument); -throw new Error('Live fixed-trace replay is disabled pending an evaluator-owned execution-contract review'); -// This exclusive create happens before source inspection, credentials, -// provider construction, or dispatch. Never unlink it: an empty file is the -// truthful crash/incomplete marker if later setup fails. -const outputReservation = reserveFixedTraceDiagnosticOutput(outputPath); - -const gitCommit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); -const gitDirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim().length > 0; -const sources = sourceBundle(); -const promptConfigVersion = sha256(JSON.stringify({ - codeVersion: CODE_VERSION, - routerRulesHash: computeRouterRulesHash(), - rules: loadRules(), - responseStyle: loadResponseStyle(), -})); -const toolDefinitions = canonicalFixedTraceToolDefinitions(traceSuite); -const budget = new FixedTraceBudget(softMaxUsd); -const plans = providerPlans(providerNames, budget); -const runStartedAt = new Date().toISOString(); -const runRootId = `fixed-trace-${runStartedAt}-${randomUUID()}`; -const artifact = await runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: sources.sha256, - gitCommit, - gitDirty, - promptConfigVersion, - traceSuite, - traceSuiteSha256: fixedTraceSuiteSha256(traceSuite), - toolDefinitions, - toolDefinitionProvenance: 'fixture_local', - architectureArm, - ...(architectureArm === 'deterministic_policy_llm_fallback_hybrid' - ? { hybridPolicy: fixedTraceHybridPolicy() } - : {}), - }, - budget, - outputReservation, - runRootId, - runStartedAt, - sourceBundleFiles: sources.files, - budgetNote: 'Soft admission target: exact prepared-request bytes and the full output allowance are reserved before each dispatch. Remote work may continue after a client timeout; any dispatched call without terminal usage marks exposure unknown and blocks every later dispatch.', -}); -console.log(JSON.stringify({ - outputPath, - runRootId, - providers: providerNames, - suite: suiteName, - comparisonEligible: artifact.comparisonEligible, - rolloutPass: artifact.rolloutPass, - budget: artifact.budget, -}, null, 2)); + dispatchable: false, + outputWritten: false, + providerCalls: 0, + externalFinalN: estimate.externalFinalN, + totalCeilingUsd: estimate.totalCeilingUsd, + }), +); diff --git a/server/tests/unit/addie/fixed-trace-budget.test.ts b/server/tests/unit/addie/fixed-trace-budget.test.ts index bc3be356e3..732465c97c 100644 --- a/server/tests/unit/addie/fixed-trace-budget.test.ts +++ b/server/tests/unit/addie/fixed-trace-budget.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it, vi } from "vitest"; import { BudgetedFixedTraceProvider, FixedTraceBudget, @@ -6,8 +6,8 @@ import { fixedTraceEstimatedCostUsd, fixedTraceApprovedPricingProfiles, fixedTraceResponsePricingPolicy, -} from '../../../src/addie/eval/fixed-trace-budget.js'; -import { collectModelResponse } from '../../../src/addie/model-providers/events.js'; +} from "../../../src/addie/eval/fixed-trace-budget.js"; +import { collectModelResponse } from "../../../src/addie/model-providers/events.js"; import type { ModelProvider, ModelProviderCapabilities, @@ -16,7 +16,7 @@ import type { ModelResponse, NormalizedModelEvent, PreparedModelInvocation, -} from '../../../src/addie/model-providers/model-provider.js'; +} from "../../../src/addie/model-providers/model-provider.js"; const CAPABILITIES: ModelProviderCapabilities = { streaming: false, @@ -30,42 +30,45 @@ const CAPABILITIES: ModelProviderCapabilities = { }; const REQUEST: ModelRequest = { - model: 'gpt-5.6-luna', + model: "gpt-5.6-luna", system: [], - messages: [{ role: 'user', content: [{ type: 'text', text: 'Synthetic request.' }] }], + messages: [ + { role: "user", content: [{ type: "text", text: "Synthetic request." }] }, + ], tools: [], maxOutputTokens: 100, }; const RESPONSE: ModelResponse = { - provider: 'openai', - model: 'gpt-5.6-luna', - id: 'response-1', - content: [{ type: 'text', text: 'Synthetic response.' }], - finishReason: 'stop', - providerFinishReason: 'completed', + provider: "openai", + model: "gpt-5.6-luna", + id: "response-1", + content: [{ type: "text", text: "Synthetic response." }], + finishReason: "stop", + providerFinishReason: "completed", usage: { inputTokens: 10, outputTokens: 5 }, }; const PRICING = { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', + profileId: "openai-gpt-5.6-luna-2026-08-26", inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset' as const, - cacheWriteAccounting: 'unsupported' as const, - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', + cacheReadAccounting: "subset" as const, + cacheWriteAccounting: "unsupported" as const, + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", }; const RESPONSE_PRICING_POLICY = fixedTraceResponsePricingPolicy( - 'openai', - 'gpt-5.6-luna', + "openai", + "gpt-5.6-luna", PRICING, ); class BudgetScriptedProvider implements ModelProvider { - readonly id = 'openai' as const; + readonly id = "openai" as const; readonly capabilities = CAPABILITIES; readonly dispatches = vi.fn(); @@ -87,101 +90,144 @@ class BudgetScriptedProvider implements ModelProvider { await options.beforeDispatch?.(this.prepare(request)); this.dispatches(); const next = this.script.shift(); - if (!next) throw new Error('Script exhausted'); + if (!next) throw new Error("Script exhausted"); if (next instanceof Error) throw next; - yield { type: 'response_start', provider: this.id, model: next.model, id: next.id }; - yield { type: 'text_delta', index: 0, text: 'Synthetic response.' }; - yield { type: 'response_complete', response: next }; + yield { + type: "response_start", + provider: this.id, + model: next.model, + id: next.id, + }; + yield { type: "text_delta", index: 0, text: "Synthetic response." }; + yield { type: "response_complete", response: next }; } } -describe('fixed trace provider budget', () => { - it('exposes only reviewed production pricing and rejects former test profiles before dispatch', () => { +describe("fixed trace provider budget", () => { + it("exposes only reviewed production pricing and rejects former test profiles before dispatch", () => { const liveProfiles = fixedTraceApprovedPricingProfiles(); expect(liveProfiles).toHaveLength(4); for (const profile of liveProfiles) { - expect(`${profile.expectedModel}\n${profile.profileId}\n${profile.source}`).not.toMatch(/synthetic|test/i); + expect( + `${profile.expectedModel}\n${profile.profileId}\n${profile.source}`, + ).not.toMatch(/synthetic|test/i); } const delegate = new BudgetScriptedProvider([RESPONSE]); - expect(() => fixedTraceResponsePricingPolicy('anthropic', 'synthetic-manual-model', { - profileId: 'synthetic-manual-artifact-v1', - inputUsdPerMillionTokens: 1, - outputUsdPerMillionTokens: 5, - cacheReadUsdPerMillionTokens: null, - cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'unsupported', - cacheWriteAccounting: 'unsupported', - source: 'Synthetic manual artifact pricing.', - })).toThrow('Fixed trace pricing profile is not evaluator approved'); + expect(() => + fixedTraceResponsePricingPolicy("anthropic", "synthetic-manual-model", { + profileId: "synthetic-manual-artifact-v1", + inputUsdPerMillionTokens: 1, + outputUsdPerMillionTokens: 5, + cacheReadUsdPerMillionTokens: null, + cacheWriteUsdPerMillionTokens: null, + cacheReadAccounting: "unsupported", + cacheWriteAccounting: "unsupported", + source: "Synthetic manual artifact pricing.", + }), + ).toThrow("Fixed trace pricing profile is not evaluator approved"); expect(delegate.dispatches).not.toHaveBeenCalled(); }); - it('prices Google-style subset reads plus additive writes explicitly', () => { - expect(fixedTraceEstimatedCostUsd({ inputTokens: 100, outputTokens: 10, cacheReadTokens: 40, cacheWriteTokens: 20 }, { - ...PRICING, - cacheReadUsdPerMillionTokens: 0.5, - cacheWriteUsdPerMillionTokens: 1, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'additive', - })).toBeCloseTo(0.00015); + it("prices Google-style subset reads plus additive writes explicitly", () => { + expect( + fixedTraceEstimatedCostUsd( + { + inputTokens: 100, + outputTokens: 10, + cacheReadTokens: 40, + cacheWriteTokens: 20, + }, + { + ...PRICING, + cacheReadUsdPerMillionTokens: 0.5, + cacheWriteUsdPerMillionTokens: 1, + cacheReadAccounting: "subset", + cacheWriteAccounting: "additive", + }, + ), + ).toBeCloseTo(0.00015); }); - it('prices additive Anthropic cache buckets without treating them as input subsets', () => { - expect(fixedTraceEstimatedCostUsd({ - inputTokens: 10, - outputTokens: 0, - cacheReadTokens: 20, - cacheWriteTokens: 30, - }, { - ...PRICING, - cacheReadUsdPerMillionTokens: 2, - cacheWriteUsdPerMillionTokens: 3, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - })).toBeCloseTo(0.00014); + it("prices additive Anthropic cache buckets without treating them as input subsets", () => { + expect( + fixedTraceEstimatedCostUsd( + { + inputTokens: 10, + outputTokens: 0, + cacheReadTokens: 20, + cacheWriteTokens: 30, + }, + { + ...PRICING, + cacheReadUsdPerMillionTokens: 2, + cacheWriteUsdPerMillionTokens: 3, + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + }, + ), + ).toBeCloseTo(0.00014); }); - it('fails closed when a nonzero cache bucket has no recorded formula', () => { - expect(() => fixedTraceEstimatedCostUsd({ inputTokens: 10, outputTokens: 0, cacheReadTokens: 1 }, { - ...PRICING, - cacheReadUsdPerMillionTokens: null, - cacheReadAccounting: 'unsupported', - })) - .toThrow('cache read accounting is unavailable'); + it("fails closed when a nonzero cache bucket has no recorded formula", () => { + expect(() => + fixedTraceEstimatedCostUsd( + { inputTokens: 10, outputTokens: 0, cacheReadTokens: 1 }, + { + ...PRICING, + cacheReadUsdPerMillionTokens: null, + cacheReadAccounting: "unsupported", + }, + ), + ).toThrow("cache read accounting is unavailable"); }); - it('closes shared admission rather than settling an unapproved returned model at requested rates', async () => { - const mismatched = { ...RESPONSE, model: 'other-openai-model' }; + it("closes shared admission rather than settling an unapproved returned model at requested rates", async () => { + const mismatched = { ...RESPONSE, model: "other-openai-model" }; const delegate = new BudgetScriptedProvider([mismatched, RESPONSE]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).resolves.toEqual(mismatched); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).resolves.toEqual(mismatched); expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0, dispatchedCalls: 1, completedCalls: 0, exposureUnknown: true, }); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toMatchObject({ - name: 'FixedTraceBudgetAdmissionError', reason: 'budget_exposure_unknown', + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toMatchObject({ + name: "FixedTraceBudgetAdmissionError", + reason: "budget_exposure_unknown", }); expect(delegate.dispatches).toHaveBeenCalledTimes(1); }); - it('rejects a caller callback as returned-model pricing authority', () => { + it("rejects a caller callback as returned-model pricing authority", () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(1); - expect(() => new BudgetedFixedTraceProvider( - delegate, - budget, - PRICING, - (() => true) as unknown as typeof RESPONSE_PRICING_POLICY, - )).toThrow('Fixed trace returned-model pricing policy is not evaluator approved'); + expect( + () => + new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + (() => true) as unknown as typeof RESPONSE_PRICING_POLICY, + ), + ).toThrow( + "Fixed trace returned-model pricing policy is not evaluator approved", + ); }); - it('reserves an additive cache-write worst case before dispatch', () => { + it("reserves an additive cache-write worst case before dispatch", () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(10_000); const reservation = budget.reserve(delegate.prepare(REQUEST), 1, { @@ -189,24 +235,31 @@ describe('fixed trace provider budget', () => { outputUsdPerMillionTokens: 0, cacheReadUsdPerMillionTokens: null, cacheWriteUsdPerMillionTokens: 1_000_000, - cacheReadAccounting: 'unsupported', - cacheWriteAccounting: 'additive', - source: 'synthetic additive cache-write worst case', + cacheReadAccounting: "unsupported", + cacheWriteAccounting: "additive", + source: "synthetic additive cache-write worst case", }); // The cache-write rate is deliberately much larger than input. A reserve // that only charged base input would be zero here. expect(budget.snapshot().reservedUsd).toBeGreaterThan(1); budget.cancel(reservation); }); - it('rejects over-budget work before provider dispatch', async () => { + it("rejects over-budget work before provider dispatch", async () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(0.000001); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toMatchObject({ - name: 'FixedTraceBudgetAdmissionError', - reason: 'soft_limit_exceeded', - terminalStatus: 'not_dispatched_budget', + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toMatchObject({ + name: "FixedTraceBudgetAdmissionError", + reason: "soft_limit_exceeded", + terminalStatus: "not_dispatched_budget", }); expect(delegate.dispatches).not.toHaveBeenCalled(); expect(budget.snapshot()).toMatchObject({ @@ -219,12 +272,19 @@ describe('fixed trace provider budget', () => { }); }); - it('releases the reserve and accounts terminal usage', async () => { + it("releases the reserve and accounts terminal usage", async () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).resolves.toEqual(RESPONSE); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).resolves.toEqual(RESPONSE); expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0.000008, reservedUsd: 0, @@ -235,31 +295,46 @@ describe('fixed trace provider budget', () => { }); }); - it('uses one frozen terminal snapshot despite delegate mutation after response_complete', async () => { + it("uses one frozen terminal snapshot despite delegate mutation after response_complete", async () => { const original = structuredClone(RESPONSE); const delegate: ModelProvider = { - id: 'openai', + id: "openai", capabilities: CAPABILITIES, prepare(request): PreparedModelInvocation { return { - provider: 'openai', model: request.model, capabilities: CAPABILITIES, + provider: "openai", + model: request.model, + capabilities: CAPABILITIES, providerRequest: { model: request.model }, }; }, - async *respond(request, options = {}): AsyncIterable { + async *respond( + request, + options = {}, + ): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); - yield { type: 'response_start', provider: 'openai', model: original.model, id: original.id }; - yield { type: 'text_delta', index: 0, text: 'Synthetic response.' }; - yield { type: 'response_complete', response: original }; - original.id = 'mutated-response-id'; - original.model = 'mutated-model'; - original.content[0] = { type: 'text', text: 'Mutated response.' }; + yield { + type: "response_start", + provider: "openai", + model: original.model, + id: original.id, + }; + yield { type: "text_delta", index: 0, text: "Synthetic response." }; + yield { type: "response_complete", response: original }; + original.id = "mutated-response-id"; + original.model = "mutated-model"; + original.content[0] = { type: "text", text: "Mutated response." }; original.usage.inputTokens = 999_999; original.usage.outputTokens = 999_999; }, }; const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); const collected = await collectModelResponse(provider.respond(REQUEST)); @@ -275,35 +350,52 @@ describe('fixed trace provider budget', () => { }); }); - it('halts later calls after a dispatched response has unknown usage', async () => { - const delegate = new BudgetScriptedProvider([new Error('transport failed'), RESPONSE]); + it("halts later calls after a dispatched response has unknown usage", async () => { + const delegate = new BudgetScriptedProvider([ + new Error("transport failed"), + RESPONSE, + ]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toThrow('transport failed'); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toThrow("transport failed"); expect(budget.snapshot()).toMatchObject({ dispatchedCalls: 1, completedCalls: 0, exposureUnknown: true, }); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toBeInstanceOf( - FixedTraceBudgetAdmissionError, - ); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toBeInstanceOf(FixedTraceBudgetAdmissionError); expect(delegate.dispatches).toHaveBeenCalledTimes(1); expect(budget.snapshot().budgetRejectedCalls).toBe(1); }); - it('treats malformed terminal usage as unknown exposure', async () => { - const delegate = new BudgetScriptedProvider([{ - ...RESPONSE, - usage: { inputTokens: -1, outputTokens: 5 }, - }]); + it("treats malformed terminal usage as unknown exposure", async () => { + const delegate = new BudgetScriptedProvider([ + { + ...RESPONSE, + usage: { inputTokens: -1, outputTokens: 5 }, + }, + ]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); - - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toThrow( - 'Fixed trace budget usage is invalid', + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, ); + + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toThrow("Fixed trace budget usage is invalid"); expect(budget.snapshot()).toMatchObject({ reservedUsd: 0, remainingUsd: null, @@ -313,14 +405,25 @@ describe('fixed trace provider budget', () => { }); }); - it('does not mark exposure unknown when the caller hook blocks dispatch', async () => { + it("does not mark exposure unknown when the caller hook blocks dispatch", async () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST, { - beforeDispatch: () => { throw new Error('local policy rejected'); }, - }))).rejects.toThrow('local policy rejected'); + await expect( + collectModelResponse( + provider.respond(REQUEST, { + beforeDispatch: () => { + throw new Error("local policy rejected"); + }, + }), + ), + ).rejects.toThrow("local policy rejected"); expect(delegate.dispatches).not.toHaveBeenCalled(); expect(budget.snapshot()).toMatchObject({ reservedUsd: 0, diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts index d63376a115..cafcd8ed56 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts @@ -1,59 +1,89 @@ -import { describe, expect, it } from 'vitest'; -import { execFileSync } from 'node:child_process'; -import { existsSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { parseFixedTraceDiagnosticCliArguments } from '../../../src/addie/eval/fixed-trace-diagnostic-cli.js'; +import { describe, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { parseFixedTraceDiagnosticCliArguments } from "../../../src/addie/eval/fixed-trace-diagnostic-cli.js"; -describe('fixed-trace diagnostic CLI parser', () => { - it('accepts only bounded dry-run forms', () => { - expect(parseFixedTraceDiagnosticCliArguments(['--validate-only', '--providers=openai'])) - .toEqual({ validateOnly: true, providers: 'openai', architectureArm: undefined, suite: undefined, softMaxUsd: undefined, output: undefined, experimentPlan: undefined, trustedManifest: undefined }); - expect(parseFixedTraceDiagnosticCliArguments(['--validate-only=true']).validateOnly).toBe(true); +describe("fixed-trace diagnostic CLI parser", () => { + it("accepts only bounded dry-run forms", () => { + expect( + parseFixedTraceDiagnosticCliArguments([ + "--validate-only", + "--providers=openai", + ]), + ).toEqual({ + validateOnly: true, + providers: "openai", + architectureArm: undefined, + suite: undefined, + softMaxUsd: undefined, + output: undefined, + experimentPlan: undefined, + trustedManifest: undefined, + }); + expect( + parseFixedTraceDiagnosticCliArguments(["--validate-only=true"]) + .validateOnly, + ).toBe(true); }); it.each([ - ['--validate-only=false'], ['--validate-onl'], ['--providers=openai', '--providers=google'], - ['positional'], ['--judge-providers=openai'], ['--providers'], ['--suite=unknown'], - ])('rejects unsafe option input %j', (args) => { + ["--validate-only=false"], + ["--validate-onl"], + ["--providers=openai", "--providers=google"], + ["positional"], + ["--judge-providers=openai"], + ["--providers"], + ["--suite=unknown"], + ])("rejects unsafe option input %j", (args) => { expect(() => parseFixedTraceDiagnosticCliArguments(args)).toThrow(); }); - it('validates a complete bare dry run without credentials, writes, or provider setup', () => { - const output = resolve('/tmp/fixed-trace-diagnostic-cli-no-write.json'); - const result = execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=direct_generation', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, OPENAI_API_KEY: '' } }); - const validated = result.split('\n').map((line) => { - try { return JSON.parse(line) as Record; } catch { return null; } - }).find((line) => line?.diagnosticOnly === true); + it("validates without credentials, writes, provider setup, or dispatch", () => { + const result = execFileSync( + "npx", + [ + "tsx", + "server/tests/manual/fixed-trace-provider-eval.ts", + "--validate-only", + ], + { + cwd: process.cwd(), + encoding: "utf8", + env: { PATH: process.env.PATH ?? "" }, + }, + ); + const validated = result + .split("\n") + .map((line) => { + try { + return JSON.parse(line) as Record; + } catch { + return null; + } + }) + .find((line) => line?.diagnosticOnly === true); expect(validated).toMatchObject({ diagnosticOnly: true, - validated: { providers: ['openai'], architectureArm: 'direct_generation', suite: 'canonical', softMaxUsd: 1, outputPath: output }, + dispatchable: false, + outputWritten: false, + providerCalls: 0, }); - expect(existsSync(output)).toBe(false); - }, 20_000); - - it('binds the reviewed hybrid evaluator suite only to the hybrid arm during validate-only planning', () => { - const output = resolve('/tmp/fixed-trace-diagnostic-cli-hybrid-suite-no-write.json'); - const result = execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=deterministic_policy_llm_fallback_hybrid', '--suite=hybrid-evaluator', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, OPENAI_API_KEY: '' } }); - expect(result).toContain('"suite":"hybrid-evaluator"'); - expect(() => execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=two_stage_llm_router', '--suite=hybrid-evaluator', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); - expect(existsSync(output)).toBe(false); }, 20_000); - it.each([ - ['--soft-max-usd=0', '--output=/tmp/out.json'], - ['--experiment-plan=/tmp/no-plan.json'], - ])('rejects malformed dry run configuration', (...args) => { - expect(() => execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', ...args, - ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); - }); + it.each([["--output=/tmp/out.json"], ["--validate-only=false"]])( + "rejects malformed dry run configuration", + (...args) => { + expect(() => + execFileSync( + "npx", + [ + "tsx", + "server/tests/manual/fixed-trace-provider-eval.ts", + "--validate-only", + ...args, + ], + { cwd: process.cwd(), stdio: "pipe" }, + ), + ).toThrow(); + }, + ); }); diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts index 4a900f3901..6c1a31f341 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts @@ -1,26 +1,26 @@ -import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; import { BudgetedFixedTraceProvider, FixedTraceBudget, fixedTraceResponsePricingPolicy, -} from '../../../src/addie/eval/fixed-trace-budget.js'; +} from "../../../src/addie/eval/fixed-trace-budget.js"; import { assertFixedTraceDiagnosticBudgetReconciliation, runFixedTraceDiagnosticArtifact, type FixedTraceDiagnosticProviderPlan, -} from '../../../src/addie/eval/fixed-trace-diagnostic-run.js'; -import { reserveFixedTraceDiagnosticOutput } from '../../../src/addie/eval/fixed-trace-diagnostic-output.js'; -import { canonicalFixedTraceToolDefinitions } from '../../../src/addie/eval/fixed-trace-tools.js'; -import { fixedTraceHybridPolicy } from '../../../src/addie/eval/fixed-trace-architecture.js'; -import type { FixedTraceProviderStageConfig } from '../../../src/addie/eval/fixed-trace-runner.js'; +} from "../../../src/addie/eval/fixed-trace-diagnostic-run.js"; +import { reserveFixedTraceDiagnosticOutput } from "../../../src/addie/eval/fixed-trace-diagnostic-output.js"; +import { canonicalFixedTraceToolDefinitions } from "../../../src/addie/eval/fixed-trace-tools.js"; +import { fixedTraceHybridPolicy } from "../../../src/addie/eval/fixed-trace-architecture.js"; +import type { FixedTraceProviderStageConfig } from "../../../src/addie/eval/fixed-trace-runner.js"; import { FIXED_TRACE_SUITE, fixedTraceSuiteSha256, type FixedTracePricing, -} from '../../../src/addie/eval/fixed-trace-suite.js'; +} from "../../../src/addie/eval/fixed-trace-suite.js"; import type { ModelProvider, ModelProviderCapabilities, @@ -30,13 +30,13 @@ import type { ModelResponse, NormalizedModelEvent, PreparedModelInvocation, -} from '../../../src/addie/model-providers/model-provider.js'; +} from "../../../src/addie/model-providers/model-provider.js"; const CAPABILITIES: ModelProviderCapabilities = { streaming: false, structuredOutput: true, reasoning: true, - reasoningEfforts: ['none'], + reasoningEfforts: ["none"], customTools: true, providerWebSearch: false, imageInput: false, @@ -44,57 +44,66 @@ const CAPABILITIES: ModelProviderCapabilities = { }; const PRICING: FixedTracePricing = { - profileId: 'anthropic-standard-2026-08:claude-haiku-4-5', + profileId: "anthropic-standard-2026-08:claude-haiku-4-5", inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, cacheReadUsdPerMillionTokens: 0.1, cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.', + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + source: + "Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.", }; -const MODEL = 'claude-haiku-4-5'; -const OPENAI_MODEL = 'gpt-5.6-luna'; +const MODEL = "claude-haiku-4-5"; +const OPENAI_MODEL = "gpt-5.6-luna"; const OPENAI_PRICING: FixedTracePricing = { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', + profileId: "openai-gpt-5.6-luna-2026-08-26", inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'unsupported', - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', + cacheReadAccounting: "subset", + cacheWriteAccounting: "unsupported", + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", }; const ZERO_RATE_PRICING: FixedTracePricing = { ...PRICING, - profileId: 'synthetic-zero-rate-artifact-v1', + profileId: "synthetic-zero-rate-artifact-v1", inputUsdPerMillionTokens: 0, outputUsdPerMillionTokens: 0, - source: 'Synthetic zero-rate artifact pricing.', + source: "Synthetic zero-rate artifact pricing.", }; const DIAGNOSTIC_TEST_REQUEST: ModelRequest = { model: MODEL, system: [], - messages: [{ role: 'user', content: [{ type: 'text', text: 'Synthetic request.' }] }], + messages: [ + { role: "user", content: [{ type: "text", text: "Synthetic request." }] }, + ], tools: [], maxOutputTokens: 1, }; function scriptedRouter( afterFinalResponse?: (response: ModelResponse) => void, - providerId: ModelProviderId = 'anthropic', + providerId: ModelProviderId = "anthropic", ): { provider: ModelProvider; calls: ModelRequest[]; response: ModelResponse } { const calls: ModelRequest[] = []; const response: ModelResponse = { provider: providerId, - model: providerId === 'openai' ? OPENAI_MODEL : MODEL, + model: providerId === "openai" ? OPENAI_MODEL : MODEL, id: `${providerId}-scripted-router-ignore`, - content: [{ type: 'text', text: JSON.stringify({ action: 'ignore', reason: 'Synthetic route.' }) }], - finishReason: 'stop', - providerFinishReason: 'stop', + content: [ + { + type: "text", + text: JSON.stringify({ action: "ignore", reason: "Synthetic route." }), + }, + ], + finishReason: "stop", + providerFinishReason: "stop", usage: { inputTokens: 10, outputTokens: 5 }, }; const provider: ModelProvider = { @@ -106,15 +115,30 @@ function scriptedRouter( model: request.model, capabilities: CAPABILITIES, requestMetadata: request.requestMetadata, - providerRequest: structuredClone(request) as unknown as Readonly>, + providerRequest: structuredClone(request) as unknown as Readonly< + Record + >, }; }, - async *respond(request: ModelRequest, options: ModelRespondOptions = {}): AsyncIterable { + async *respond( + request: ModelRequest, + options: ModelRespondOptions = {}, + ): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); calls.push(structuredClone(request)); - yield { type: 'response_start', provider: providerId, model: response.model, id: response.id }; - yield { type: 'text_delta', index: 0, text: response.content[0].type === 'text' ? response.content[0].text : '' }; - yield { type: 'response_complete', response }; + yield { + type: "response_start", + provider: providerId, + model: response.model, + id: response.id, + }; + yield { + type: "text_delta", + index: 0, + text: + response.content[0].type === "text" ? response.content[0].text : "", + }; + yield { type: "response_complete", response }; afterFinalResponse?.(response); }, }; @@ -131,29 +155,55 @@ function cloneChangingIdentityProvider(): { const provider: ModelProvider = { get id(): ModelProviderId { reads++; - return reads === 1 ? 'anthropic' : 'openai'; + return reads === 1 ? "anthropic" : "openai"; }, capabilities: CAPABILITIES, prepare(request): PreparedModelInvocation { // The delegate's request surface is stable; only an old clone's second // read of `id` would change the wrapper identity. return { - provider: 'anthropic', model: request.model, capabilities: CAPABILITIES, + provider: "anthropic", + model: request.model, + capabilities: CAPABILITIES, requestMetadata: request.requestMetadata, - providerRequest: structuredClone(request) as unknown as Readonly>, + providerRequest: structuredClone(request) as unknown as Readonly< + Record + >, }; }, async *respond(request, options = {}): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); calls.push(structuredClone(request)); const response: ModelResponse = { - provider: 'anthropic', model: MODEL, id: 'stable-response', - content: [{ type: 'text', text: JSON.stringify({ action: 'ignore', reason: 'Synthetic route.' }) }], - finishReason: 'stop', providerFinishReason: 'stop', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "stable-response", + content: [ + { + type: "text", + text: JSON.stringify({ + action: "ignore", + reason: "Synthetic route.", + }), + }, + ], + finishReason: "stop", + providerFinishReason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, + }; + yield { + type: "response_start", + provider: "anthropic", + model: response.model, + id: response.id, + }; + yield { + type: "text_delta", + index: 0, + text: + response.content[0].type === "text" ? response.content[0].text : "", }; - yield { type: 'response_start', provider: 'anthropic', model: response.model, id: response.id }; - yield { type: 'text_delta', index: 0, text: response.content[0].type === 'text' ? response.content[0].text : '' }; - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; }, }; return { provider, calls, idReads: () => reads }; @@ -161,17 +211,19 @@ function cloneChangingIdentityProvider(): { function stage( provider: ModelProvider, - pricing: FixedTracePricing = provider.id === 'openai' ? OPENAI_PRICING : PRICING, + pricing: FixedTracePricing = provider.id === "openai" + ? OPENAI_PRICING + : PRICING, ): FixedTraceProviderStageConfig { return { provider, - model: provider.id === 'openai' ? OPENAI_MODEL : MODEL, - reasoningEffort: 'none', + model: provider.id === "openai" ? OPENAI_MODEL : MODEL, + reasoningEffort: "none", maxOutputTokens: 300, timeoutMs: 30_000, maxIterations: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', + samplingMode: "provider_no_sampling_control", temperature: null, pricing: structuredClone(pricing), }; @@ -180,7 +232,9 @@ function stage( function budgetedStage( provider: ModelProvider, budget: FixedTraceBudget, - pricing: FixedTracePricing = provider.id === 'openai' ? OPENAI_PRICING : PRICING, + pricing: FixedTracePricing = provider.id === "openai" + ? OPENAI_PRICING + : PRICING, ): FixedTraceProviderStageConfig { const configured = stage(provider, pricing); return { @@ -189,92 +243,156 @@ function budgetedStage( provider, budget, configured.pricing, - fixedTraceResponsePricingPolicy(provider.id, configured.model, configured.pricing), + fixedTraceResponsePricingPolicy( + provider.id, + configured.model, + configured.pricing, + ), ), }; } -function twoTurnProvider( - afterRouterResponse?: () => void, -): { provider: ModelProvider; calls: ModelRequest[] } { +function twoTurnProvider(afterRouterResponse?: () => void): { + provider: ModelProvider; + calls: ModelRequest[]; +} { const calls: ModelRequest[] = []; let generationTurn = 0; const provider: ModelProvider = { - id: 'anthropic', + id: "anthropic", capabilities: CAPABILITIES, prepare(request): PreparedModelInvocation { return { - provider: 'anthropic', model: request.model, capabilities: CAPABILITIES, + provider: "anthropic", + model: request.model, + capabilities: CAPABILITIES, requestMetadata: request.requestMetadata, - providerRequest: structuredClone(request) as unknown as Readonly>, + providerRequest: structuredClone(request) as unknown as Readonly< + Record + >, }; }, async *respond(request, options = {}): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); calls.push(structuredClone(request)); - const router = request.requestMetadata?.purpose === 'fixed_trace_router'; + const router = request.requestMetadata?.purpose === "fixed_trace_router"; const response: ModelResponse = router ? { - provider: 'anthropic', model: MODEL, id: 'router', - content: [{ type: 'text', text: JSON.stringify({ - action: 'respond', tool_sets: ['knowledge'], confidence: 'high', - requires_depth: false, reason: 'Synthetic route.', - }) }], - finishReason: 'stop', providerFinishReason: 'stop', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "router", + content: [ + { + type: "text", + text: JSON.stringify({ + action: "respond", + tool_sets: ["knowledge"], + confidence: "high", + requires_depth: false, + reason: "Synthetic route.", + }), + }, + ], + finishReason: "stop", + providerFinishReason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, } : generationTurn++ === 0 ? { - provider: 'anthropic', model: MODEL, id: 'generation-tool', - content: [{ type: 'tool_call', id: 'tool-1', name: 'search_docs', input: { query: 'task model' } }], - finishReason: 'tool_calls', providerFinishReason: 'tool_use', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "generation-tool", + content: [ + { + type: "tool_call", + id: "tool-1", + name: "search_docs", + input: { query: "task model" }, + }, + ], + finishReason: "tool_calls", + providerFinishReason: "tool_use", + usage: { inputTokens: 10, outputTokens: 5 }, } : { - provider: 'anthropic', model: MODEL, id: 'generation-final', - content: [{ type: 'text', text: 'A buyer calls a seller task and receives its structured response.' }], - finishReason: 'stop', providerFinishReason: 'stop', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "generation-final", + content: [ + { + type: "text", + text: "A buyer calls a seller task and receives its structured response.", + }, + ], + finishReason: "stop", + providerFinishReason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, }; - yield { type: 'response_start', provider: 'anthropic', model: response.model, id: response.id }; + yield { + type: "response_start", + provider: "anthropic", + model: response.model, + id: response.id, + }; for (const [index, content] of response.content.entries()) { - if (content.type === 'text') yield { type: 'text_delta', index, text: content.text }; - if (content.type === 'tool_call') yield { type: 'tool_call', index, call: content }; + if (content.type === "text") + yield { type: "text_delta", index, text: content.text }; + if (content.type === "tool_call") + yield { type: "tool_call", index, call: content }; } - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; if (router) afterRouterResponse?.(); }, }; return { provider, calls }; } -describe('fixed-trace diagnostic output reservation', () => { - it('never overwrites an existing artifact', () => { - const path = join(mkdtempSync(join(tmpdir(), 'fixed-trace-output-')), 'artifact.json'); - writeFileSync(path, 'existing'); - expect(() => reserveFixedTraceDiagnosticOutput(path)).toThrow('Cannot exclusively reserve'); - expect(readFileSync(path, 'utf8')).toBe('existing'); +describe("fixed-trace diagnostic output reservation", () => { + it("never overwrites an existing artifact", () => { + const path = join( + mkdtempSync(join(tmpdir(), "fixed-trace-output-")), + "artifact.json", + ); + writeFileSync(path, "existing"); + expect(() => reserveFixedTraceDiagnosticOutput(path)).toThrow( + "Cannot exclusively reserve", + ); + expect(readFileSync(path, "utf8")).toBe("existing"); }); - it('rejects directory and missing-parent targets before dispatch', () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - expect(() => reserveFixedTraceDiagnosticOutput(directory)).toThrow('Cannot exclusively reserve'); - expect(() => reserveFixedTraceDiagnosticOutput(join(directory, 'missing', 'artifact.json'))).toThrow('Cannot exclusively reserve'); + it("rejects directory and missing-parent targets before dispatch", () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + expect(() => reserveFixedTraceDiagnosticOutput(directory)).toThrow( + "Cannot exclusively reserve", + ); + expect(() => + reserveFixedTraceDiagnosticOutput( + join(directory, "missing", "artifact.json"), + ), + ).toThrow("Cannot exclusively reserve"); }); - it('claims then finalizes through one exclusive descriptor', () => { - const path = join(mkdtempSync(join(tmpdir(), 'fixed-trace-output-')), 'artifact.json'); + it("claims then finalizes through one exclusive descriptor", () => { + const path = join( + mkdtempSync(join(tmpdir(), "fixed-trace-output-")), + "artifact.json", + ); const reservation = reserveFixedTraceDiagnosticOutput(path); reservation.finalize('{"diagnosticOnly":true}\n'); - expect(readFileSync(path, 'utf8')).toBe('{"diagnosticOnly":true}\n'); + expect(readFileSync(path, "utf8")).toBe('{"diagnosticOnly":true}\n'); }); - it('runs the manual diagnostic candidate path into a complete reserved artifact with scripted providers', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("runs the manual diagnostic candidate path into a complete reserved artifact with scripted providers", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const plan: FixedTraceDiagnosticProviderPlan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; @@ -282,26 +400,26 @@ describe('fixed-trace diagnostic output reservation', () => { const artifact = await runFixedTraceDiagnosticArtifact({ plans: [plan], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), - gitCommit: 'abcdef0', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', + promptConfigVersion: "synthetic-manual-prompt-v1", traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', - architectureArm: 'deterministic_policy_llm_fallback_hybrid', + toolDefinitionProvenance: "fixture_local", + architectureArm: "deterministic_policy_llm_fallback_hybrid", hybridPolicy: fixedTraceHybridPolicy(), }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); - const persisted = JSON.parse(readFileSync(path, 'utf8')) as typeof artifact; + const persisted = JSON.parse(readFileSync(path, "utf8")) as typeof artifact; expect(router.calls).toHaveLength(1); expect(persisted).toMatchObject({ complete: true, @@ -309,19 +427,28 @@ describe('fixed-trace diagnostic output reservation', () => { comparisonEligible: false, promotionEvidenceEligible: false, rolloutPass: false, - architectureArm: { id: 'deterministic_policy_llm_fallback_hybrid', diagnosticOnly: true }, + architectureArm: { + id: "deterministic_policy_llm_fallback_hybrid", + diagnosticOnly: true, + }, hybridPolicy: fixedTraceHybridPolicy(), - runs: [{ - provider: 'anthropic', - summary: { complete: true, comparisonEligible: false }, - observations: [{ traceId: selectedTrace.id, terminalStatus: 'ignored' }], - }], + runs: [ + { + provider: "anthropic", + summary: { complete: true, comparisonEligible: false }, + observations: [ + { traceId: selectedTrace.id, terminalStatus: "ignored" }, + ], + }, + ], }); expect(persisted.runs[0].observations).toHaveLength(1); - expect(artifact.runs[0].runId).toBe('synthetic-manual-root:anthropic'); - expect(artifact.runs[0].observations.every((observation) => ( - observation.metadata.runId === artifact.runs[0].runId - ))).toBe(true); + expect(artifact.runs[0].runId).toBe("synthetic-manual-root:anthropic"); + expect( + artifact.runs[0].observations.every( + (observation) => observation.metadata.runId === artifact.runs[0].runId, + ), + ).toBe(true); expect(artifact.budget).toMatchObject({ accountedSpendUsd: 0.000035, dispatchedCalls: 1, @@ -335,40 +462,44 @@ describe('fixed-trace diagnostic output reservation', () => { estimatedCostUsd: 0.000035, }); expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBe(0.000035); - expect(persisted.runs[0].observations[0].metadata.router.usage).toMatchObject({ inputTokens: 10 }); + expect( + persisted.runs[0].observations[0].metadata.router.usage, + ).toMatchObject({ inputTokens: 10 }); }); - it('freezes the complete two-plan artifact contract before a provider can mutate later plans', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); - const sourceBundleFiles = ['before.ts']; + it("freezes the complete two-plan artifact contract before a provider can mutate later plans", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); + const sourceBundleFiles = ["before.ts"]; const budget = new FixedTraceBudget(1); const baseConfig = { - sourceBundleSha256: 'a'.repeat(64), - gitCommit: 'abcdef0', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", gitDirty: false, - promptConfigVersion: 'before-prompt', + promptConfigVersion: "before-prompt", traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local' as const, - architectureArm: 'two_stage_llm_router' as const, + toolDefinitionProvenance: "fixture_local" as const, + architectureArm: "two_stage_llm_router" as const, }; let secondPlan!: FixedTraceDiagnosticProviderPlan; const first = scriptedRouter(() => { - baseConfig.sourceBundleSha256 = 'b'.repeat(64); - baseConfig.promptConfigVersion = 'forged-after-first-plan'; - sourceBundleFiles.push('forged-after-first-plan.ts'); + baseConfig.sourceBundleSha256 = "b".repeat(64); + baseConfig.promptConfigVersion = "forged-after-first-plan"; + sourceBundleFiles.push("forged-after-first-plan.ts"); secondPlan.router.maxOutputTokens = 1; - secondPlan.router.pricing.source = 'forged-after-first-plan'; + secondPlan.router.pricing.source = "forged-after-first-plan"; }); const second = scriptedRouter(() => { first.response.usage.inputTokens = 999_999; - }, 'openai'); + }, "openai"); secondPlan = { - name: 'openai', + name: "openai", router: budgetedStage(second.provider, budget), generation: budgetedStage(second.provider, budget), }; @@ -376,7 +507,7 @@ describe('fixed-trace diagnostic output reservation', () => { const artifact = await runFixedTraceDiagnosticArtifact({ plans: [ { - name: 'anthropic', + name: "anthropic", router: budgetedStage(first.provider, budget), generation: budgetedStage(first.provider, budget), }, @@ -385,27 +516,30 @@ describe('fixed-trace diagnostic output reservation', () => { baseConfig, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", sourceBundleFiles, - budgetNote: 'Synthetic no-network budget note.', + budgetNote: "Synthetic no-network budget note.", }); - const persisted = JSON.parse(readFileSync(path, 'utf8')) as typeof artifact; + const persisted = JSON.parse(readFileSync(path, "utf8")) as typeof artifact; expect(persisted).toMatchObject({ - sourceBundleSha256: 'a'.repeat(64), - promptConfigVersion: 'before-prompt', - sourceBundleFiles: ['before.ts'], - requestedProviders: ['anthropic', 'openai'], + sourceBundleSha256: "a".repeat(64), + promptConfigVersion: "before-prompt", + sourceBundleFiles: ["before.ts"], + requestedProviders: ["anthropic", "openai"], }); expect(artifact.runs.map((run) => run.runId)).toEqual([ - 'synthetic-manual-root:anthropic', - 'synthetic-manual-root:openai', + "synthetic-manual-root:anthropic", + "synthetic-manual-root:openai", ]); expect(persisted.runs[1].requestedConfig.router).toMatchObject({ - provider: 'openai', + provider: "openai", maxOutputTokens: 300, - pricing: { source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.' }, + pricing: { + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", + }, }); expect(persisted.runs[0].observations[0].metadata.router).toMatchObject({ usage: { inputTokens: 10, outputTokens: 5 }, @@ -422,58 +556,71 @@ describe('fixed-trace diagnostic output reservation', () => { } }); - it('rejects a provider-mismatched plan before scripted dispatch', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a provider-mismatched plan before scripted dispatch", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const baseConfig = { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local' as const, - architectureArm: 'two_stage_llm_router' as const, + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local" as const, + architectureArm: "two_stage_llm_router" as const, }; - const invoke = (plans: FixedTraceDiagnosticProviderPlan[]) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', - }); + const invoke = (plans: FixedTraceDiagnosticProviderPlan[]) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); const plan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; - await expect(invoke([{ ...plan, name: 'not-anthropic' }])).rejects.toThrow('provider plans require unique names'); - const duplicatePath = join(directory, 'duplicate-artifact.json'); - await expect(runFixedTraceDiagnosticArtifact({ - plans: [plan, { ...plan }], - baseConfig, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(duplicatePath), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', - })).rejects.toThrow('provider plans require unique names'); + await expect(invoke([{ ...plan, name: "not-anthropic" }])).rejects.toThrow( + "provider plans require unique names", + ); + const duplicatePath = join(directory, "duplicate-artifact.json"); + await expect( + runFixedTraceDiagnosticArtifact({ + plans: [plan, { ...plan }], + baseConfig, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(duplicatePath), + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }), + ).rejects.toThrow("provider plans require unique names"); expect(router.calls).toHaveLength(0); - expect(readFileSync(path, 'utf8')).toBe(''); - expect(readFileSync(duplicatePath, 'utf8')).toBe(''); + expect(readFileSync(path, "utf8")).toBe(""); + expect(readFileSync(duplicatePath, "utf8")).toBe(""); }); - it('rejects a plan identity accessor before it can change validation into execution', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'failed-artifact.json'); - const completedPath = join(directory, 'completed-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a plan identity accessor before it can change validation into execution", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "failed-artifact.json"); + const completedPath = join(directory, "completed-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); let nameReads = 0; @@ -483,176 +630,308 @@ describe('fixed-trace diagnostic output reservation', () => { // before either a lease or a provider dispatch is possible. get name() { nameReads++; - return nameReads <= 6 ? 'anthropic' : 'forged'; + return nameReads <= 6 ? "anthropic" : "forged"; }, router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; - const invoke = (plans: readonly FixedTraceDiagnosticProviderPlan[], path: string) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - }); + const invoke = ( + plans: readonly FixedTraceDiagnosticProviderPlan[], + path: string, + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([accessorPlan] as unknown as FixedTraceDiagnosticProviderPlan[], failedPath)) - .rejects.toThrow('provider plan 0.name must be an own data property'); + await expect( + invoke( + [accessorPlan] as unknown as FixedTraceDiagnosticProviderPlan[], + failedPath, + ), + ).rejects.toThrow("provider plan 0.name must be an own data property"); expect(nameReads).toBe(0); expect(router.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); + expect(readFileSync(failedPath, "utf8")).toBe(""); expect(budget.snapshot()).toMatchObject({ - accountedSpendUsd: 0, reservedUsd: 0, dispatchedCalls: 0, - completedCalls: 0, budgetRejectedCalls: 0, admissionClosed: false, exposureUnknown: false, + accountedSpendUsd: 0, + reservedUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 0, + admissionClosed: false, + exposureUnknown: false, }); - const artifact = await invoke([{ - name: 'anthropic', router: accessorPlan.router, generation: accessorPlan.generation, - }], completedPath); + const artifact = await invoke( + [ + { + name: "anthropic", + router: accessorPlan.router, + generation: accessorPlan.generation, + }, + ], + completedPath, + ); expect(router.calls).toHaveLength(1); - expect(artifact.runs[0]).toMatchObject({ provider: 'anthropic', runId: 'root:anthropic' }); + expect(artifact.runs[0]).toMatchObject({ + provider: "anthropic", + runId: "root:anthropic", + }); }); - it('never rereads a delegate identity while cloning an authenticated plan', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("never rereads a delegate identity while cloning an authenticated plan", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const changing = cloneChangingIdentityProvider(); const budget = new FixedTraceBudget(1); const wrapper = new BudgetedFixedTraceProvider( changing.provider, budget, PRICING, - fixedTraceResponsePricingPolicy('anthropic', MODEL, PRICING), + fixedTraceResponsePricingPolicy("anthropic", MODEL, PRICING), ); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: stage(wrapper), generation: stage(wrapper) }], + plans: [ + { + name: "anthropic", + router: stage(wrapper), + generation: stage(wrapper), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(changing.idReads()).toBe(1); expect(changing.calls).toHaveLength(1); - expect(artifact).toMatchObject({ requestedProviders: ['anthropic'] }); - expect(artifact.runs[0]).toMatchObject({ provider: 'anthropic', runId: 'root:anthropic' }); + expect(artifact).toMatchObject({ requestedProviders: ["anthropic"] }); + expect(artifact.runs[0]).toMatchObject({ + provider: "anthropic", + runId: "root:anthropic", + }); expect(artifact.runs[0].observations[0].metadata.router).toMatchObject({ - requestedProvider: 'anthropic', returnedProvider: 'anthropic', estimatedCostUsd: 0.000035, + requestedProvider: "anthropic", + returnedProvider: "anthropic", + estimatedCostUsd: 0.000035, }); }); - it('rejects a self-declared zero-rate profile before lease or dispatch and leaves the budget reusable', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'forged-artifact.json'); - const retryPath = join(directory, 'retry-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a self-declared zero-rate profile before lease or dispatch and leaves the budget reusable", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "forged-artifact.json"); + const retryPath = join(directory, "retry-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1e-12); const trustedRouter = budgetedStage(router.provider, budget); const trustedGeneration = budgetedStage(router.provider, budget); const forgedPricing: FixedTracePricing = { ...ZERO_RATE_PRICING, - profileId: 'attacker-says-reviewed-v1', - source: 'attacker assertion', + profileId: "attacker-says-reviewed-v1", + source: "attacker assertion", }; - const invoke = (plans: readonly FixedTraceDiagnosticProviderPlan[], path: string) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - }); + const invoke = ( + plans: readonly FixedTraceDiagnosticProviderPlan[], + path: string, + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([{ - name: 'anthropic', router: { ...trustedRouter, pricing: forgedPricing }, generation: trustedGeneration, - }], failedPath)).rejects.toThrow('Fixed trace pricing profile is not evaluator approved'); + await expect( + invoke( + [ + { + name: "anthropic", + router: { ...trustedRouter, pricing: forgedPricing }, + generation: trustedGeneration, + }, + ], + failedPath, + ), + ).rejects.toThrow("Fixed trace pricing profile is not evaluator approved"); expect(router.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); + expect(readFileSync(failedPath, "utf8")).toBe(""); expect(budget.snapshot()).toMatchObject({ - accountedSpendUsd: 0, reservedUsd: 0, dispatchedCalls: 0, - completedCalls: 0, budgetRejectedCalls: 0, admissionClosed: false, exposureUnknown: false, + accountedSpendUsd: 0, + reservedUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 0, + admissionClosed: false, + exposureUnknown: false, }); - const artifact = await invoke([{ - name: 'anthropic', router: trustedRouter, generation: trustedGeneration, - }], retryPath); - expect(artifact.budget).toMatchObject({ dispatchedCalls: 0, completedCalls: 0, budgetRejectedCalls: 1 }); - expect(readFileSync(retryPath, 'utf8')).not.toBe(''); + const artifact = await invoke( + [ + { + name: "anthropic", + router: trustedRouter, + generation: trustedGeneration, + }, + ], + retryPath, + ); + expect(artifact.budget).toMatchObject({ + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 1, + }); + expect(readFileSync(retryPath, "utf8")).not.toBe(""); }); - it('rejects nested pricing accessors without reading them and leaves the budget reusable', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'accessor-artifact.json'); - const retryPath = join(directory, 'retry-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects nested pricing accessors without reading them and leaves the budget reusable", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "accessor-artifact.json"); + const retryPath = join(directory, "retry-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const trustedRouter = budgetedStage(router.provider, budget); const trustedGeneration = budgetedStage(router.provider, budget); const accessorPricing = { ...PRICING } as FixedTracePricing; let pricingReads = 0; - Object.defineProperty(accessorPricing, 'inputUsdPerMillionTokens', { + Object.defineProperty(accessorPricing, "inputUsdPerMillionTokens", { enumerable: true, - get() { pricingReads++; return 0; }, - }); - const invoke = (plans: readonly FixedTraceDiagnosticProviderPlan[], path: string) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + get() { + pricingReads++; + return 0; }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', }); + const invoke = ( + plans: readonly FixedTraceDiagnosticProviderPlan[], + path: string, + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([{ - name: 'anthropic', router: { ...trustedRouter, pricing: accessorPricing }, generation: trustedGeneration, - }], failedPath)).rejects.toThrow('provider plan 0.router.pricing.inputUsdPerMillionTokens must be an own data property'); + await expect( + invoke( + [ + { + name: "anthropic", + router: { ...trustedRouter, pricing: accessorPricing }, + generation: trustedGeneration, + }, + ], + failedPath, + ), + ).rejects.toThrow( + "provider plan 0.router.pricing.inputUsdPerMillionTokens must be an own data property", + ); expect(pricingReads).toBe(0); expect(router.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); - expect(budget.snapshot()).toMatchObject({ dispatchedCalls: 0, completedCalls: 0, budgetRejectedCalls: 0 }); + expect(readFileSync(failedPath, "utf8")).toBe(""); + expect(budget.snapshot()).toMatchObject({ + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 0, + }); - await invoke([{ name: 'anthropic', router: trustedRouter, generation: trustedGeneration }], retryPath); + await invoke( + [ + { + name: "anthropic", + router: trustedRouter, + generation: trustedGeneration, + }, + ], + retryPath, + ); expect(router.calls).toHaveLength(1); }); - it('does not let a final-response mutation alter its snapshotted manual artifact', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("does not let a final-response mutation alter its snapshotted manual artifact", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); let plan!: FixedTraceDiagnosticProviderPlan; - const router = scriptedRouter(() => { plan.router.model = 'mutated-after-final-response'; }); + const router = scriptedRouter(() => { + plan.router.model = "mutated-after-final-response"; + }); const budget = new FixedTraceBudget(1); plan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; @@ -660,62 +939,84 @@ describe('fixed-trace diagnostic output reservation', () => { const artifact = await runFixedTraceDiagnosticArtifact({ plans: [plan], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), - gitCommit: 'abcdef0', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', + promptConfigVersion: "synthetic-manual-prompt-v1", traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', - architectureArm: 'two_stage_llm_router', + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(router.calls).toHaveLength(1); expect(artifact.runs[0].requestedConfig.router.model).toBe(MODEL); - expect(JSON.parse(readFileSync(path, 'utf8'))).toMatchObject({ complete: true, diagnosticOnly: true }); + expect(JSON.parse(readFileSync(path, "utf8"))).toMatchObject({ + complete: true, + diagnosticOnly: true, + }); }); - it('derives child run IDs internally instead of accepting an unrelated callback result', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("derives child run IDs internally instead of accepting an unrelated callback result", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget) }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, // This former input is intentionally ignored at runtime as well as // removed from the public type, so a JavaScript caller cannot forge it. - runIdForProvider: () => 'unrelated-id', + runIdForProvider: () => "unrelated-id", budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", } as unknown as Parameters[0]); - expect(artifact.runs[0].runId).toBe('root:anthropic'); - expect(artifact.runs[0].observations[0].metadata.runId).toBe('root:anthropic'); + expect(artifact.runs[0].runId).toBe("root:anthropic"); + expect(artifact.runs[0].observations[0].metadata.runId).toBe( + "root:anthropic", + ); }); - it('rejects a subclass that claims budget binding while bypassing the ledger', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a subclass that claims budget binding while bypassing the ledger", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const delegate = scriptedRouter(); const budget = new FixedTraceBudget(1); class BypassingBudgetProvider extends BudgetedFixedTraceProvider { @@ -724,13 +1025,15 @@ describe('fixed-trace diagnostic output reservation', () => { delegate.provider, budget, PRICING, - fixedTraceResponsePricingPolicy('anthropic', MODEL, PRICING), + fixedTraceResponsePricingPolicy("anthropic", MODEL, PRICING), ); } // This was previously trusted through instanceof plus a public, // overridable isBoundToBudget predicate. - isBoundToBudget(): boolean { return true; } + isBoundToBudget(): boolean { + return true; + } override async *respond( request: ModelRequest, @@ -741,51 +1044,85 @@ describe('fixed-trace diagnostic output reservation', () => { } const bypass = new BypassingBudgetProvider(); - await expect(runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: stage(bypass), generation: stage(bypass) }], - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - })).rejects.toThrow('provider plans require unique names'); + await expect( + runFixedTraceDiagnosticArtifact({ + plans: [ + { + name: "anthropic", + router: stage(bypass), + generation: stage(bypass), + }, + ], + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }), + ).rejects.toThrow("provider plans require unique names"); expect(delegate.calls).toHaveLength(0); - expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0, dispatchedCalls: 0, completedCalls: 0 }); - expect(readFileSync(path, 'utf8')).toBe(''); + expect(budget.snapshot()).toMatchObject({ + accountedSpendUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + }); + expect(readFileSync(path, "utf8")).toBe(""); }); - it('keeps collector, metadata, summary, ledger, and artifact on the terminal snapshot', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("keeps collector, metadata, summary, ledger, and artifact on the terminal snapshot", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter((response) => { - response.id = 'forged-id'; - response.model = 'forged-model'; - response.content[0] = { type: 'text', text: 'forged response' }; + response.id = "forged-id"; + response.model = "forged-model"; + response.content[0] = { type: "text", text: "forged response" }; response.usage.inputTokens = 999_999; response.usage.outputTokens = 999_999; }); const budget = new FixedTraceBudget(1); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget) }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); - const persisted = JSON.parse(readFileSync(path, 'utf8')) as typeof artifact; + const persisted = JSON.parse(readFileSync(path, "utf8")) as typeof artifact; const observation = artifact.runs[0].observations[0]; expect(observation.metadata.router).toMatchObject({ @@ -794,111 +1131,183 @@ describe('fixed-trace diagnostic output reservation', () => { estimatedCostUsd: 0.000035, }); expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBe(0.000035); - expect(artifact.budget).toMatchObject({ accountedSpendUsd: 0.000035, dispatchedCalls: 1, completedCalls: 1 }); - expect(persisted.runs[0].observations[0].metadata.router.usage).toMatchObject({ inputTokens: 10, outputTokens: 5 }); + expect(artifact.budget).toMatchObject({ + accountedSpendUsd: 0.000035, + dispatchedCalls: 1, + completedCalls: 1, + }); + expect( + persisted.runs[0].observations[0].metadata.router.usage, + ).toMatchObject({ inputTokens: 10, outputTokens: 5 }); expect(persisted.budget.accountedSpendUsd).toBe(0.000035); }); - it('retains an unknown-model response as unknown exposure without inventing a spend equality', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("retains an unknown-model response as unknown exposure without inventing a spend equality", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); - router.response.model = 'unapproved-model'; + router.response.model = "unapproved-model"; const budget = new FixedTraceBudget(1); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ - name: 'anthropic', - router: budgetedStage(router.provider, budget), - generation: budgetedStage(router.provider, budget), - }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(artifact.runs[0].observations[0].metadata.router).toMatchObject({ - source: 'provider', returnedModel: 'unapproved-model', estimatedCostUsd: null, + source: "provider", + returnedModel: "unapproved-model", + estimatedCostUsd: null, }); expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBeNull(); expect(artifact.budget).toMatchObject({ - accountedSpendUsd: 0, dispatchedCalls: 1, completedCalls: 0, exposureUnknown: true, + accountedSpendUsd: 0, + dispatchedCalls: 1, + completedCalls: 0, + exposureUnknown: true, }); }); - it('rejects a settled ledger for an unpriced dispatched provider response', () => { + it("rejects a settled ledger for an unpriced dispatched provider response", () => { const providerStage = { - source: 'provider', dispatched: true, dispatchedCalls: 1, - usageKnown: true, usage: { inputTokens: 10, outputTokens: 5 }, estimatedCostUsd: null, + source: "provider", + dispatched: true, + dispatchedCalls: 1, + usageKnown: true, + usage: { inputTokens: 10, outputTokens: 5 }, + estimatedCostUsd: null, }; const notRunStage = { - source: 'not_run', dispatched: false, dispatchedCalls: 0, - usageKnown: false, usage: null, estimatedCostUsd: 0, + source: "not_run", + dispatched: false, + dispatchedCalls: 0, + usageKnown: false, + usage: null, + estimatedCostUsd: 0, }; - expect(() => assertFixedTraceDiagnosticBudgetReconciliation({ - policy: 'soft_admission_target', softMaxUsd: 1, accountedSpendUsd: 0.000035, - reservedUsd: 0, remainingUsd: 0.999965, dispatchedCalls: 1, completedCalls: 1, - budgetRejectedCalls: 0, admissionClosed: false, exposureUnknown: false, - }, [{ observations: [{ - terminalStatus: 'complete', - metadata: { router: providerStage, generation: notRunStage }, - }] }] as never)).toThrow('unpriced dispatched response lacks unknown budget exposure'); + expect(() => + assertFixedTraceDiagnosticBudgetReconciliation( + { + policy: "soft_admission_target", + softMaxUsd: 1, + accountedSpendUsd: 0.000035, + reservedUsd: 0, + remainingUsd: 0.999965, + dispatchedCalls: 1, + completedCalls: 1, + budgetRejectedCalls: 0, + admissionClosed: false, + exposureUnknown: false, + }, + [ + { + observations: [ + { + terminalStatus: "complete", + metadata: { router: providerStage, generation: notRunStage }, + }, + ], + }, + ] as never, + ), + ).toThrow("unpriced dispatched response lacks unknown budget exposure"); }); - it('reconciles a pre-dispatch budget rejection with a local/not-run observation', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("reconciles a pre-dispatch budget rejection with a local/not-run observation", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(0.000001); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget) }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(artifact.runs[0].observations[0]).toMatchObject({ - terminalStatus: 'not_dispatched_budget', - metadata: { router: { source: 'local', dispatched: false }, generation: { source: 'not_run' } }, + terminalStatus: "not_dispatched_budget", + metadata: { + router: { source: "local", dispatched: false }, + generation: { source: "not_run" }, + }, }); expect(artifact.budget).toMatchObject({ - accountedSpendUsd: 0, dispatchedCalls: 0, completedCalls: 0, budgetRejectedCalls: 1, exposureUnknown: false, + accountedSpendUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 1, + exposureUnknown: false, }); }); - it('preflights every plan before leasing a pristine budget or dispatching an earlier plan', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'failed-artifact.json'); - const completedPath = join(directory, 'completed-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); - const first = scriptedRouter(undefined, 'anthropic'); - const second = scriptedRouter(undefined, 'openai'); + it("preflights every plan before leasing a pristine budget or dispatching an earlier plan", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "failed-artifact.json"); + const completedPath = join(directory, "completed-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); + const first = scriptedRouter(undefined, "anthropic"); + const second = scriptedRouter(undefined, "openai"); const budget = new FixedTraceBudget(1); const firstPlan: FixedTraceDiagnosticProviderPlan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(first.provider, budget), generation: budgetedStage(first.provider, budget), }; const invalidSecondPlan: FixedTraceDiagnosticProviderPlan = { - name: 'openai', + name: "openai", router: budgetedStage(second.provider, budget), generation: budgetedStage(second.provider, budget), }; @@ -906,26 +1315,34 @@ describe('fixed-trace diagnostic output reservation', () => { const invoke = ( plans: readonly FixedTraceDiagnosticProviderPlan[], path: string, - ) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - }); + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([firstPlan, invalidSecondPlan], failedPath)).rejects.toThrow( - 'generation maxIterations must be between', - ); + await expect( + invoke([firstPlan, invalidSecondPlan], failedPath), + ).rejects.toThrow("generation maxIterations must be between"); expect(first.calls).toHaveLength(0); expect(second.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); + expect(readFileSync(failedPath, "utf8")).toBe(""); expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0, reservedUsd: 0, @@ -937,7 +1354,7 @@ describe('fixed-trace diagnostic output reservation', () => { }); const validSecondPlan: FixedTraceDiagnosticProviderPlan = { - name: 'openai', + name: "openai", router: budgetedStage(second.provider, budget), generation: budgetedStage(second.provider, budget), }; @@ -953,93 +1370,177 @@ describe('fixed-trace diagnostic output reservation', () => { expect(artifact.budget.accountedSpendUsd).toBeCloseTo(0.000043); }); - it('prevents post-preflight method and prototype tampering in a two-turn zero-rate run', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'knowledge-task-model'); - if (!selectedTrace) throw new Error('Missing synthetic knowledge trace'); + it("prevents post-preflight method and prototype tampering in a two-turn zero-rate run", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "knowledge-task-model", + ); + if (!selectedTrace) throw new Error("Missing synthetic knowledge trace"); const attacks: string[] = []; let generation!: BudgetedFixedTraceProvider; const delegate = twoTurnProvider(() => { const replace = (name: string, attempt: () => void) => { - try { attempt(); } catch { attacks.push(name); } + try { + attempt(); + } catch { + attacks.push(name); + } }; - replace('own_respond', () => Object.defineProperty(generation, 'respond', { value: delegate.provider.respond })); - replace('own_prepare', () => Object.defineProperty(generation, 'prepare', { value: delegate.provider.prepare })); - replace('prototype_swap', () => Object.setPrototypeOf(generation, {})); - replace('prototype_respond', () => Object.defineProperty(BudgetedFixedTraceProvider.prototype, 'respond', { value: delegate.provider.respond })); - replace('prototype_prepare', () => Object.defineProperty(BudgetedFixedTraceProvider.prototype, 'prepare', { value: delegate.provider.prepare })); + replace("own_respond", () => + Object.defineProperty(generation, "respond", { + value: delegate.provider.respond, + }), + ); + replace("own_prepare", () => + Object.defineProperty(generation, "prepare", { + value: delegate.provider.prepare, + }), + ); + replace("prototype_swap", () => Object.setPrototypeOf(generation, {})); + replace("prototype_respond", () => + Object.defineProperty(BudgetedFixedTraceProvider.prototype, "respond", { + value: delegate.provider.respond, + }), + ); + replace("prototype_prepare", () => + Object.defineProperty(BudgetedFixedTraceProvider.prototype, "prepare", { + value: delegate.provider.prepare, + }), + ); }); const budget = new FixedTraceBudget(1); - const policy = fixedTraceResponsePricingPolicy('anthropic', MODEL, PRICING); - const router = new BudgetedFixedTraceProvider(delegate.provider, budget, PRICING, policy); - generation = new BudgetedFixedTraceProvider(delegate.provider, budget, PRICING, policy); + const policy = fixedTraceResponsePricingPolicy("anthropic", MODEL, PRICING); + const router = new BudgetedFixedTraceProvider( + delegate.provider, + budget, + PRICING, + policy, + ); + generation = new BudgetedFixedTraceProvider( + delegate.provider, + budget, + PRICING, + policy, + ); const generationStage = stage(generation, PRICING); generationStage.maxIterations = 2; const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: stage(router, PRICING), generation: generationStage }], + plans: [ + { + name: "anthropic", + router: stage(router, PRICING), + generation: generationStage, + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), - toolDefinitions: canonicalFixedTraceToolDefinitions().filter((tool) => ['search_docs', 'get_doc'].includes(tool.name)), - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + toolDefinitions: canonicalFixedTraceToolDefinitions().filter((tool) => + ["search_docs", "get_doc"].includes(tool.name), + ), + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); - expect(attacks).toEqual(['own_respond', 'own_prepare', 'prototype_swap', 'prototype_respond', 'prototype_prepare']); + expect(attacks).toEqual([ + "own_respond", + "own_prepare", + "prototype_swap", + "prototype_respond", + "prototype_prepare", + ]); expect(delegate.calls).toHaveLength(3); expect(artifact.runs[0].observations[0].metadata).toMatchObject({ router: { dispatchedCalls: 1, estimatedCostUsd: 0.000035 }, generation: { dispatchedCalls: 2, estimatedCostUsd: 0.00007 }, }); - expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBeCloseTo(0.000105); + expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBeCloseTo( + 0.000105, + ); expect(artifact.budget).toMatchObject({ - dispatchedCalls: 3, completedCalls: 3, budgetRejectedCalls: 0, exposureUnknown: false, + dispatchedCalls: 3, + completedCalls: 3, + budgetRejectedCalls: 0, + exposureUnknown: false, }); expect(artifact.budget.accountedSpendUsd).toBeCloseTo(0.000105); }); - it('rejects a zero-rate ledger with preexisting completed, unknown, or rejected activity', async () => { - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a zero-rate ledger with preexisting completed, unknown, or rejected activity", async () => { + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const invoke = async (budget: FixedTraceBudget, suffix: string) => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); const provider = scriptedRouter(); - await expect(runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(provider.provider, budget), generation: budgetedStage(provider.provider, budget) }], - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(join(directory, `${suffix}.json`)), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - })).rejects.toThrow('budget must be pristine and exclusively claimed'); + await expect( + runFixedTraceDiagnosticArtifact({ + plans: [ + { + name: "anthropic", + router: budgetedStage(provider.provider, budget), + generation: budgetedStage(provider.provider, budget), + }, + ], + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput( + join(directory, `${suffix}.json`), + ), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }), + ).rejects.toThrow("budget must be pristine and exclusively claimed"); expect(provider.calls).toHaveLength(0); }; const prepared = scriptedRouter().provider.prepare(DIAGNOSTIC_TEST_REQUEST); const completed = new FixedTraceBudget(1); - const completedReservation = completed.reserve(prepared, 1, ZERO_RATE_PRICING); + const completedReservation = completed.reserve( + prepared, + 1, + ZERO_RATE_PRICING, + ); completed.markDispatched(completedReservation); - completed.complete(completedReservation, { inputTokens: 1, outputTokens: 1 }, ZERO_RATE_PRICING); - await invoke(completed, 'completed'); + completed.complete( + completedReservation, + { inputTokens: 1, outputTokens: 1 }, + ZERO_RATE_PRICING, + ); + await invoke(completed, "completed"); const unknown = new FixedTraceBudget(1); const unknownReservation = unknown.reserve(prepared, 1, ZERO_RATE_PRICING); unknown.markDispatched(unknownReservation); unknown.markExposureUnknown(unknownReservation); - await invoke(unknown, 'unknown'); + await invoke(unknown, "unknown"); const rejected = new FixedTraceBudget(0.000001); expect(() => rejected.reserve(prepared, 1, PRICING)).toThrow(); - await invoke(rejected, 'rejected'); + await invoke(rejected, "rejected"); }); }); diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index 75d582398f..c95b931e44 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -1,338 +1,284 @@ -import { describe, expect, it } from 'vitest'; -import { createHash } from 'node:crypto'; -import { fixedTraceEstimatedCostUsd } from '../../../src/addie/eval/fixed-trace-budget.js'; -import { FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL, FIXED_TRACE_CONFIRMATORY_POWER_GATE, FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertFixedTraceEvaluationProtocol, assertFixedTraceEvaluationProtocolTrusted, estimateFixedTraceEvaluationProtocol, evaluateFixedTraceConfirmatoryClaim, fixedTraceEvaluationProtocolFingerprint, fixedTraceEvaluationProtocolRunnerBinding } from '../../../src/addie/eval/fixed-trace-evaluation-protocol.js'; -import { OPENAI_GPT_5_6_LUNA_PRICING_VERSION, resolveModelCostPricing } from '../../../src/addie/model-cost-pricing.js'; -import { snapshotFixedTraceJson } from '../../../src/addie/eval/fixed-trace-safe-snapshot.js'; +import { describe, expect, it } from "vitest"; +import { + FIXED_TRACE_ADMITTED_CELLS, + FIXED_TRACE_CONFIRMATORY_POWER_GATE, + FIXED_TRACE_CONFIRMATORY_ADMISSION, + FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT, + FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS, + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + FIXED_TRACE_PROTOCOL_PRICING, + FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, + assertPromotionGradeDualJudgeFeasibility, + assertFixedTraceEvaluationProtocol, + estimateFixedTraceEvaluationProtocol, + providerExcludingCalibratedJudges, + selectFixedTraceScreeningSurvivors, + semanticJudgeCandidateProviders, +} from "../../../src/addie/eval/fixed-trace-evaluation-protocol.js"; +import { + FIXED_TRACE_PARTITION_MANIFEST, + assertFixedTracePartitionManifest, +} from "../../../src/addie/eval/fixed-trace-partition.js"; +import { resolveModelCostPricing } from "../../../src/addie/model-cost-pricing.js"; -function historicalOwnEnumerableFingerprint(value: unknown): string { - const canonical = (current: unknown): string => { - if (current === null || typeof current === 'boolean' || typeof current === 'string' || typeof current === 'number') return JSON.stringify(current); - if (Array.isArray(current)) return `[${current.map(canonical).join(',')}]`; - if (typeof current === 'object') return `{${Object.keys(current).sort().map((key) => `${JSON.stringify(key)}:${canonical((current as Record)[key])}`).join(',')}}`; - throw new Error('not JSON'); - }; - return createHash('sha256').update(canonical(value), 'utf8').digest('hex'); -} - -describe('fixed-trace evaluation protocol projection', () => { - it('is ordered, diagnostic-only, non-dispatchable, and non-promotional', () => { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); - assertFixedTraceEvaluationProtocol(protocol); - expect(protocol.phases.map((phase) => phase.id)).toEqual(['bounded_smoke', 'router_screen', 'oracle_generator_ceiling', 'deployable_architecture', 'controlled_tuning']); - expect(protocol.unavailableFinalTarget).toEqual({ availability: 'unavailable', uniqueCaseCount: 38, repetitions: 3, missingCaseCount: 38 }); - expect(protocol.phases.every((phase) => phase.resultUse === 'diagnostic_only')).toBe(true); - expect(estimateFixedTraceEvaluationProtocol(protocol)).toMatchObject({ - dispatchable: false, - expectedSpendUsd: null, - budgetProjection: { - screeningTuning: { uniqueEvaluableCaseCount: 120, approvalCeilingUsd: null }, - confirmatory: { requiredIndependentEvaluableCaseCount: 10_562, unavailableTargetCaseCount: 38, approvalCeilingUsd: null }, +describe("fixed-trace staged protocol", () => { + it("derives the complete 46 development / 36 tuning partitions from corpus authority", () => { + assertFixedTracePartitionManifest(); + expect(FIXED_TRACE_PARTITION_MANIFEST.development).toHaveLength(46); + expect(FIXED_TRACE_PARTITION_MANIFEST.tuning).toHaveLength(36); + expect( + new Set([ + ...FIXED_TRACE_PARTITION_MANIFEST.development, + ...FIXED_TRACE_PARTITION_MANIFEST.tuning, + ]).size, + ).toBe(82); + }); + it("screens every reviewed adapter-supported provider/model/effort cell before adaptive pruning", () => { + for (const role of ["router", "generation"] as const) + for (const provider of ["anthropic", "openai", "google"] as const) + expect( + FIXED_TRACE_ADMITTED_CELLS.some( + (cell) => cell.role === role && cell.provider === provider, + ), + ).toBe(true); + expect( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.provider === "openai", + ).map((cell) => cell.effort), + ).toContain("high"); + expect( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.provider === "google", + ).map((cell) => cell.effort), + ).toContain("medium"); + expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.adaptiveRule).toMatchObject( + { + smokeCases: 8, + developmentCases: 46, + tuningCases: 36, + selection: "predeclared_pareto_successive_halving", + repeats: "stability_only_not_new_cases", }, - }); + ); }); - - it('labels nominal 38-case margins inconclusive and does not treat repeated generations as independent cases', () => { - expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.primaryHypothesisFamily).toEqual({ - size: 2, correction: 'holm', orderedOneSidedAlpha: [0.0125, 0.025], - }); - expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.superiorityRequiredIndependentEvaluableCases).toBe(3_803); - expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.nonInferiorityRequiredIndependentEvaluableCases).toBe(10_562); - const nominalAt38 = evaluateFixedTraceConfirmatoryClaim({ - pairedCaseIds: Array.from({ length: 38 }, (_, index) => `case-${index + 1}`), - observedSuperiorityPercentagePoints: 5.1, - observedNonInferiorityPercentagePoints: -2.9, + it("keeps hybrid router fallback and direct admission explicit in worst-case accounting", () => { + const estimate = estimateFixedTraceEvaluationProtocol(); + expect(estimate.hybridWorstCaseRouterCalls).toBe(138); + expect(estimate.hybridWorstCaseRouterCeilingUsd).toBeCloseTo(0.772248, 10); + const architecture = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + expect( + architecture.arms.find((arm) => arm.architecture === "direct_generation") + ?.admission, + ).toBe("not_admitted_architecture"); + const hybrid = architecture.arms.find( + (arm) => arm.architecture === "deterministic_policy_llm_fallback_hybrid", + )!; + expect(hybrid.stages.some((stage) => stage.role === "router")).toBe(true); + expect(hybrid.admission).toBe("not_evaluable_no_treatment_contrast"); + expect(FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT).toEqual([ + expect.objectContaining({ + phase: "development", + totalCases: 46, + localTerminalCases: 0, + routedCases: 46, + evaluable: false, + blocker: "no_hybrid_treatment_contrast", + }), + expect.objectContaining({ + phase: "tuning", + totalCases: 36, + localTerminalCases: 0, + routedCases: 36, + evaluable: false, + blocker: "no_hybrid_treatment_contrast", + }), + ]); + expect( + estimate.armCallAccounting.find( + (arm) => arm.armId === "hybrid-locked-finalist", + ), + ).toMatchObject({ + evaluable: false, + localTerminalCases: 0, + routedCases: 138, + routerCalls: 138, + generationCalls: 1_656, + routerCeilingUsd: 0.772248, }); - expect(nominalAt38).toMatchObject({ - independentEvaluableCaseCount: 38, - nominalMarginsReached: true, - confirmatoryClaim: 'refused_underpowered', - }); - - const repeatedGenerations = evaluateFixedTraceConfirmatoryClaim({ - pairedCaseIds: Array.from({ length: 38 * 3 }, (_, index) => `case-${index % 38}`), - observedSuperiorityPercentagePoints: 5.1, - observedNonInferiorityPercentagePoints: -2.9, + }); + it("has no fictional final N and binds named Holm hypotheses", () => { + expect( + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.finalProtocol, + ).toMatchObject({ + status: "unavailable", + externalPackDigest: null, + externalN: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, + fingerprint: null, + powerResult: null, }); - expect(repeatedGenerations).toMatchObject({ - independentEvaluableCaseCount: 38, - repeatedObservationCount: 76, - requiredIndependentEvaluableCaseCount: 10_562, - confirmatoryClaim: 'refused_underpowered', + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases, + ).toBe(10_562); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.superiorityRequiredIndependentEvaluableCases, + ).toBe(3_803); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.hypotheses.map( + (hypothesis) => hypothesis.id, + ), + ).toEqual(["H1-superiority", "H2-non-inferiority"]); + expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE).toMatchObject({ + targetPower: 0.8, + conservativeDiscordanceVarianceUpperBound: 1, + hypotheses: [ + { + id: "H1-superiority", + marginPercentagePoints: 0, + exactTest: "exact_conditional_mcnemar_zero_margin_only", + }, + { + id: "H2-non-inferiority", + marginPercentagePoints: -3, + exactTest: + "predeclared_exact_unconditional_matched_pair_test_required", + }, + ], }); - expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredAnalysis).toEqual({ - resampling: 'grouped_stratified_case_level_bootstrap', - multiplicityCorrection: 'holm', - pairedDiscordancePower: 'evaluator_owned_exact_paired_discordance_contract_unavailable', - pairedDiscordanceTest: 'predeclared_exact_paired_test_required', + expect(FIXED_TRACE_CONFIRMATORY_ADMISSION).toMatchObject({ + status: "not_admitted_missing_fingerprinted_statistical_protocol", + holm: { K: 2, oneSidedFamilyAlpha: 0.025 }, + unitOfAnalysis: "unique_conversation_user_episode", + repeatedAndTemplateRelatedObservationRule: + "cluster_by_conversation_user_episode; repetitions_never_increase_N", + sizingPilot: { + heldOutFromFinal: true, + reusableInFinal: false, + conservativeDiscordanceUpperBound: null, + }, + judgeCalibration: { + allowedRelationshipToScoredDevelopment: "separate_or_cross_fitted_only", + }, + finalProtocolFingerprint: null, }); }); - it('locks a same-generator, provider-excluding, two-judge architecture ablation', () => { - expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find((item) => item.id === 'router_screen')?.arms.map((arm) => arm.stages[0] && [arm.stages[0].model, arm.stages[0].reasoningEffort])) - .toEqual([['claude-haiku-4-5', 'provider_default'], ['gpt-5.6-luna', 'none']]); - expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find((item) => item.id === 'oracle_generator_ceiling')?.arms.map((arm) => arm.stages[0] && [arm.stages[0].model, arm.stages[0].reasoningEffort])) - .toEqual([['claude-sonnet-5', 'provider_default'], ['claude-haiku-4-5', 'provider_default']]); - const phase = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find((item) => item.id === 'deployable_architecture')!; - expect(phase.arms.map((arm) => arm.id)).toEqual(['routed-haiku-sonnet', 'safe-hybrid-sonnet', 'bounded-direct-sonnet']); - expect(phase.arms.map((arm) => arm.ablationControlId)).toEqual([FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id, FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id, FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL.id]); - for (const arm of phase.arms) { - const candidate = arm.stages.filter((stage) => stage.role !== 'judge'); - const judges = arm.stages.filter((stage) => stage.role === 'judge'); - expect(candidate.filter((stage) => stage.role === 'generation')).toEqual([expect.objectContaining({ provider: 'anthropic', model: 'claude-sonnet-5', reasoningEffort: 'provider_default' })]); - expect(new Set(candidate.map((stage) => stage.provider))).toEqual(new Set(['anthropic'])); - expect(judges.map((stage) => stage.provider)).toEqual(['openai', 'google']); - expect(arm.lunaJudgeCalibration).toBe('requires_verified_luna_judge_calibration'); + it("uses two provider-excluding calibrated judge families for every candidate provider", () => { + for (const provider of ["anthropic", "openai", "google"] as const) { + const judges = providerExcludingCalibratedJudges([provider]); + expect(judges).toHaveLength(2); + expect(judges.some((judge) => judge.provider === provider)).toBe(false); } - expect(phase.arms[1]?.admission).toBe('requires_verified_hybrid_contract'); - expect(phase.arms[2]?.admission).toBe('requires_verified_direct_contract'); - const estimate = estimateFixedTraceEvaluationProtocol(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); - expect(estimate.phases.find((item) => item.phaseId === 'deployable_architecture')).toMatchObject({ judgeCalls: 46 * 3 * 3 * 2 }); - expect(estimate.judgeCeilingUsd).toBeGreaterThan(0); - expect(estimate.screening.contingencyUsd).toBeGreaterThan(0); - expect(estimate.screening.totalCeilingUsd).toBe( - estimate.screening.candidateCeilingUsd + estimate.screening.judgeCeilingUsd + estimate.screening.contingencyUsd, - ); - const selfJudging = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - selfJudging.phases[3].arms[0].stages[2].provider = 'anthropic'; - expect(() => assertFixedTraceEvaluationProtocol(selfJudging)).toThrow('evaluator-owned stage configuration matrix'); - const uncalibratedLuna = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - uncalibratedLuna.phases[3].arms[0].lunaJudgeCalibration = 'not_applicable'; - expect(() => assertFixedTraceEvaluationProtocol(uncalibratedLuna)).toThrow('evaluator-owned arm matrix'); + expect(FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS).toHaveLength(2); + expect( + FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS.every( + (judge) => + judge.calibrationCorpusSha256 === null && + judge.humanLabelsSha256 === null && + judge.outcomesSha256 === null && + judge.authenticatedAdmission === null && + judge.status === "blocked_pending_authenticated_calibration", + ), + ).toBe(true); }); - it('keeps Terra and Sol as unpriced inert descriptors', () => { - expect(FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES).toEqual([{ provider: 'openai', model: 'gpt-5.6-terra', dispatchable: false, trustedPrice: null }, { provider: 'openai', model: 'gpt-5.6-sol', dispatchable: false, trustedPrice: null }]); - const terra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); - terra.phases[1].arms[0].stages[0].provider = 'openai'; terra.phases[1].arms[0].stages[0].model = 'gpt-5.6-terra'; - expect(() => assertFixedTraceEvaluationProtocol(terra)).toThrow('evaluator-owned stage configuration matrix'); + it("fails promotion-grade dual-LLM judging for a mixed router/generator pipeline", () => { + const router = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => + cell.id === "router:anthropic:claude-haiku-4-5:provider_default", + )!; + const generator = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:openai:gpt-5.6-luna:none", + )!; + expect( + semanticJudgeCandidateProviders({ + stages: [ + { role: "router", cellId: router.id }, + { role: "generation", cellId: generator.id }, + ], + } as any), + ).toEqual(["anthropic", "openai"]); + expect(() => + assertPromotionGradeDualJudgeFeasibility({ + stages: [ + { role: "router", cellId: router.id }, + { role: "generation", cellId: generator.id }, + ], + } as any), + ).toThrow("single-provider complete pipeline"); + expect(() => + semanticJudgeCandidateProviders({ + stages: [{ role: "router", cellId: router.id }], + } as any), + ).toThrow("admitted generation provider"); }); - it('reuses only the exact approved Luna provider, model, pricing, and control identity', () => { - const luna = resolveModelCostPricing('openai', 'gpt-5.6-luna'); - expect(luna).toMatchObject({ provider: 'openai', model: 'gpt-5.6-luna', version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION }); - expect(luna?.estimateCostMicros({ inputTokens: 1_000_000, outputTokens: 1_000_000 })).toBe(1_400_000); - expect(resolveModelCostPricing('openai', 'gpt-5.6-luna-20260826')).toBeNull(); - expect(resolveModelCostPricing('openai', 'gpt-5.6-terra')).toBeNull(); - expect(resolveModelCostPricing('openai', 'gpt-5.6-sol')).toBeNull(); - }); - it('rejects reversed, duplicated, direct, smoke-promotion, and fabricated trust', () => { - const reversed = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); reversed.phases.reverse(); - expect(() => assertFixedTraceEvaluationProtocol(reversed)).toThrow('exact required order'); - const direct = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); direct.phases[3].arms[0].architecture = 'direct_bounded_production_shaped'; - expect(() => assertFixedTraceEvaluationProtocol(direct)).toThrow('evaluator-owned arm matrix'); - const promotional = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; promotional.phases[0].resultUse = 'promotional'; - expect(() => assertFixedTraceEvaluationProtocol(promotional)).toThrow(); - expect(() => assertFixedTraceEvaluationProtocolTrusted(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any)).toThrow('locked'); - expect(() => fixedTraceEvaluationProtocolRunnerBinding(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, () => ({}) as any, 'bounded_smoke', [])).toThrow('locked'); - }); - - it('rejects the reported caller substitutions before estimating a budget', () => { - const substituted = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - const phase = substituted.phases[0]; - phase.arms[0].admission = 'caller_promotional'; - phase.uniqueCaseCount = 1; - phase.repetitions = 999; - phase.arms[0].stages[0].maxInvocationsPerCase = 999; - expect(() => estimateFixedTraceEvaluationProtocol(substituted)).toThrow('evaluator-owned'); - }); - - it('rejects missing, extra, reordered, and substituted available phases', () => { - const missing = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - missing.phases.splice(2, 1); - expect(() => estimateFixedTraceEvaluationProtocol(missing)).toThrow('exact required order'); - - const extra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - extra.phases.push(structuredClone(extra.phases[0])); - expect(() => estimateFixedTraceEvaluationProtocol(extra)).toThrow('exact required order'); - - const reordered = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - [reordered.phases[0], reordered.phases[1]] = [reordered.phases[1], reordered.phases[0]]; - expect(() => estimateFixedTraceEvaluationProtocol(reordered)).toThrow('exact required order'); - - const substituted = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - substituted.phases[3].arms[1] = structuredClone(substituted.phases[3].arms[0]); - expect(() => estimateFixedTraceEvaluationProtocol(substituted)).toThrow('evaluator-owned arm matrix'); - }); - - it('enforces evaluator-owned admission, result use, counts, repetitions, and stop conditions for every phase', () => { - for (let phaseIndex = 0; phaseIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.length; phaseIndex += 1) { - for (const mutate of [ - (phase: any) => { phase.uniqueCaseCount = 1; }, - (phase: any) => { phase.repetitions = 999; }, - (phase: any) => { phase.resultUse = 'caller_promotional'; }, - ]) { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - mutate(protocol.phases[phaseIndex]); - expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow('evaluator-owned phase matrix'); - } - for (let armIndex = 0; armIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms.length; armIndex += 1) { - const admission = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - admission.phases[phaseIndex].arms[armIndex].admission = 'caller_promotional'; - expect(() => estimateFixedTraceEvaluationProtocol(admission)).toThrow('evaluator-owned arm matrix'); - for (let stageIndex = 0; stageIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms[armIndex].stages.length; stageIndex += 1) { - const stopCondition = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - stopCondition.phases[phaseIndex].arms[armIndex].stages[stageIndex].maxInvocationsPerCase = 999; - expect(() => estimateFixedTraceEvaluationProtocol(stopCondition)).toThrow('evaluator-owned stage configuration matrix'); - } - } - } - }); - - it('prices only exact evaluator-owned provider, model, and execution configurations', () => { - const profile = (provider: 'anthropic' | 'google', model: string) => - FIXED_TRACE_PROTOCOL_PRICING.find((candidate) => candidate.provider === provider && candidate.model === model)!; - const haiku = profile('anthropic', 'claude-haiku-4-5'); - const sonnet = profile('anthropic', 'claude-sonnet-5'); - const gemini = profile('google', 'gemini-3.7-flash'); - const reject = (mutate: (stage: any) => void) => { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - mutate(protocol.phases[1].arms[0].stages[0]); - expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow('evaluator-owned stage configuration matrix'); - }; - - reject((stage) => { stage.provider = gemini.provider; stage.model = gemini.model; stage.pricingProfileId = gemini.profileId; }); - reject((stage) => { stage.model = sonnet.model; stage.pricingProfileId = sonnet.profileId; }); - reject((stage) => { stage.model = 'claude-haiku-4.5'; }); - reject((stage) => { stage.pricingProfileId = sonnet.profileId; }); - expect(haiku.profileId).not.toBe(gemini.profileId); - - for (let phaseIndex = 0; phaseIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.length; phaseIndex += 1) { - for (let armIndex = 0; armIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms.length; armIndex += 1) { - for (let stageIndex = 0; stageIndex < FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases[phaseIndex].arms[armIndex].stages.length; stageIndex += 1) { - for (const mutate of [ - (stage: any) => { stage.reasoningEffort = stage.reasoningEffort === 'low' ? 'medium' : 'low'; }, - (stage: any) => { stage.maxInputTokensPerInvocation += 1; }, - (stage: any) => { stage.maxOutputTokensPerInvocation += 1; }, - (stage: any) => { stage.timeoutMs += 1; }, - (stage: any) => { stage.maxInvocationsPerCase += 1; }, - (stage: any) => { stage.transportRetries = 1; }, - (stage: any) => { stage.cacheMode = 'caller_cache'; }, - (stage: any) => { stage.samplingMode = 'caller_sampling'; }, - (stage: any) => { stage.temperature = 0; }, - ]) { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - mutate(protocol.phases[phaseIndex].arms[armIndex].stages[stageIndex]); - expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow('evaluator-owned stage configuration matrix'); - } - } - } - } - }); - - it('rejects missing, extra, reordered, duplicated, and hostile stage records before pricing', () => { - const reject = (mutate: (protocol: any) => void, message = 'evaluator-owned stage configuration matrix') => { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - mutate(protocol); - expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow(message); - }; - reject((protocol) => { protocol.phases[0].arms[0].stages.pop(); }); - reject((protocol) => { protocol.phases[0].arms[0].stages.push(structuredClone(protocol.phases[0].arms[0].stages[0])); }); - reject((protocol) => { protocol.phases[0].arms[0].stages.reverse(); }); - reject((protocol) => { protocol.phases[0].arms[0].stages[1] = structuredClone(protocol.phases[0].arms[0].stages[0]); }); - - reject((protocol) => { - const stage = protocol.phases[1].arms[0].stages[0]; - const { provider: ignoredProvider, ...own } = stage; - void ignoredProvider; - protocol.phases[1].arms[0].stages[0] = Object.assign(Object.create({ provider: 'google' }), own); - }, 'plain object'); - reject((protocol) => { - Object.defineProperty(protocol.phases[1].arms[0].stages[0], 'provider', { - enumerable: true, - get() { return 'google'; }, - }); - }, 'own enumerable data'); - reject((protocol) => { protocol.phases[1].arms[0].stages[0] = new Proxy(protocol.phases[1].arms[0].stages[0], {}); }, 'Proxy'); - reject((protocol) => { Object.setPrototypeOf(protocol.phases[1].arms[0].stages[0], { provider: 'google' }); }, 'plain object'); - reject((protocol) => { - Object.defineProperty(protocol.phases[1].arms[0].stages[0], '__proto__', { enumerable: true, value: { poisoned: true } }); - }, 'dangerous prototype key'); - }); - - it('keeps prototype-shaped JSON as visible data and rejects it at every protocol fingerprint boundary', () => { - const hostile = JSON.parse('{"__proto__":{"polluted":true}}'); - const detached = snapshotFixedTraceJson(hostile, 'hostile JSON') as Record; - expect(Object.getPrototypeOf(detached)).toBe(null); - expect(Object.keys(detached)).toEqual(['__proto__']); - expect(Object.getOwnPropertyDescriptor(detached, '__proto__')?.value).toEqual({ polluted: true }); - expect(JSON.stringify(detached)).toBe('{"__proto__":{"polluted":true}}'); - expect(({} as { polluted?: boolean }).polluted).toBeUndefined(); - - for (const key of ['__proto__', 'prototype', 'constructor']) { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - Object.defineProperty(protocol, key, { enumerable: true, value: { poisoned: true } }); - expect(() => fixedTraceEvaluationProtocolFingerprint(protocol)).toThrow('dangerous prototype key'); - } + it("fails closed if a nominally promotion-grade pipeline is made mixed-provider", () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + const architecture = protocol.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + const routed = architecture.arms.find( + (arm) => arm.id === "routed-locked-finalist", + )!; + (routed.stages[0] as { cellId: string }).cellId = + "router:openai:gpt-5.6-luna:none"; + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow( + "single-provider complete pipeline", + ); }); - - it('rejects inherited keys, symbols, accessors, Proxies, array extras, and cycles without mutating the snapshot', () => { - const inherited = Object.create({ inherited: true }); - expect(() => snapshotFixedTraceJson(inherited, 'inherited')).toThrow('plain object'); - - const symbol = { safe: true }; - Object.defineProperty(symbol, Symbol('hidden'), { enumerable: true, value: true }); - expect(() => snapshotFixedTraceJson(symbol, 'symbol')).toThrow('without symbols'); - - let reads = 0; - const accessor = {}; - Object.defineProperty(accessor, 'value', { enumerable: true, get() { reads += 1; return true; } }); - expect(() => snapshotFixedTraceJson(accessor, 'accessor')).toThrow('own enumerable data'); - expect(reads).toBe(0); - expect(() => snapshotFixedTraceJson(new Proxy({}, {}), 'proxy')).toThrow('Proxy'); - - const arrayExtra: any[] & { extra?: boolean } = [true]; - arrayExtra.extra = true; - expect(() => snapshotFixedTraceJson(arrayExtra, 'array extra')).toThrow('extra array property'); - - const cycle: { self?: unknown } = {}; - cycle.self = cycle; - expect(() => snapshotFixedTraceJson(cycle, 'cycle')).toThrow('cycle'); - - const mutable = { nested: { value: 1 } }; - const detached = snapshotFixedTraceJson(mutable, 'mutable') as { nested: { value: number } }; - mutable.nested.value = 2; - expect(detached.nested.value).toBe(1); - expect(Object.isFrozen(detached)).toBe(true); - expect(Object.isFrozen(detached.nested)).toBe(true); + it("applies hard elimination and successive halving deterministically", () => { + const results = FIXED_TRACE_ADMITTED_CELLS.slice(0, 4).map( + (cell, index) => ({ + cellId: cell.id, + safetyFailures: index === 3 ? 1 : 0, + identityFailures: 0, + malformedFailures: 0, + toolLoopFailures: 0, + reliabilityFailures: index, + latencyMs: 10 - index, + costUsd: index, + }), + ); + expect(selectFixedTraceScreeningSurvivors(results)).toEqual([ + results[0]!.cellId, + results[1]!.cellId, + ]); + expect(selectFixedTraceScreeningSurvivors([...results].reverse())).toEqual([ + results[0]!.cellId, + results[1]!.cellId, + ]); }); - - it('uses a detached closed snapshot for validation, hashing, and estimates', () => { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); - const expectedFingerprint = fixedTraceEvaluationProtocolFingerprint(protocol); - const estimate = estimateFixedTraceEvaluationProtocol(protocol); - protocol.phases[1].arms[0].stages[0].maxOutputTokensPerInvocation = 999; - expect(estimate.stages.find((stage) => stage.phaseId === 'router_screen')?.outputTokenCeiling).toBe(46 * 3 * 300); - expect(Object.isFrozen(estimate)).toBe(true); - expect(Object.isFrozen(estimate.phases)).toBe(true); - expect(expectedFingerprint).toMatch(/^[a-f0-9]{64}$/); - expect(() => fixedTraceEvaluationProtocolFingerprint(protocol)).toThrow('evaluator-owned stage configuration matrix'); - - const arrayExtra = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - arrayExtra.phases.extra = true; - expect(() => assertFixedTraceEvaluationProtocol(arrayExtra)).toThrow('extra array property'); - let getterReads = 0; - const accessor = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - Object.defineProperty(accessor, 'id', { enumerable: true, get() { getterReads += 1; return 'forged'; } }); - expect(() => assertFixedTraceEvaluationProtocol(accessor)).toThrow('own enumerable data'); - expect(getterReads).toBe(0); - expect(() => assertFixedTraceEvaluationProtocol(new Proxy(protocol, {}))).toThrow('Proxy'); + it("uses canonical Luna subset-cache pricing and leaves Terra/Sol inert", () => { + const luna = FIXED_TRACE_PROTOCOL_PRICING.find( + (profile) => profile.provider === "openai", + )!; + expect(luna.cacheReadAccounting).toBe("subset"); + expect(luna.cacheReadUsdPerMillionTokens).toBe(0.02); + expect( + resolveModelCostPricing("openai", "gpt-5.6-luna")?.estimateCostMicros({ + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 1_000_000, + }), + ).toBe(20_000); + expect( + FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES.every( + (candidate) => candidate.trustedPrice === null, + ), + ).toBe(true); }); - - it('rejects inherited Anthropic-to-Google stage substitution before it can alter cost or a fingerprint', () => { - const inheritedProtocol = (provider: 'anthropic' | 'google', model: string, pricingProfileId: string) => { - const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; - const stage = protocol.phases[1].arms[0].stages[0]; - const { provider: ignoredProvider, model: ignoredModel, pricingProfileId: ignoredPricing, ...ownFields } = stage; - void ignoredProvider; void ignoredModel; void ignoredPricing; - protocol.phases[1].arms[0].stages[0] = Object.assign(Object.create({ provider, model, pricingProfileId }), ownFields); - return protocol; - }; - const anthropic = FIXED_TRACE_PROTOCOL_PRICING.find((profile) => profile.provider === 'anthropic' && profile.model === 'claude-haiku-4-5')!; - const google = FIXED_TRACE_PROTOCOL_PRICING.find((profile) => profile.provider === 'google')!; - const inheritedAnthropic = inheritedProtocol('anthropic', anthropic.model, anthropic.profileId); - const inheritedGoogle = inheritedProtocol('google', google.model, google.profileId); - expect(historicalOwnEnumerableFingerprint(inheritedGoogle)).toBe(historicalOwnEnumerableFingerprint(inheritedAnthropic)); - expect(fixedTraceEstimatedCostUsd({ inputTokens: 46 * 3 * 4_096, outputTokens: 46 * 3 * 300, cacheReadTokens: 0, cacheWriteTokens: 0 }, google)) - .not.toBe(fixedTraceEstimatedCostUsd({ inputTokens: 46 * 3 * 4_096, outputTokens: 46 * 3 * 300, cacheReadTokens: 0, cacheWriteTokens: 0 }, anthropic)); - expect(() => fixedTraceEvaluationProtocolFingerprint(inheritedAnthropic)).toThrow('plain object'); - expect(() => fixedTraceEvaluationProtocolFingerprint(inheritedGoogle)).toThrow('plain object'); + it("rejects changing the unadmitted direct boundary or final availability", () => { + const direct = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + direct.phases[4].arms[2].admission = "admitted_diagnostic"; + expect(() => assertFixedTraceEvaluationProtocol(direct)).toThrow( + "not_admitted_architecture", + ); + const final = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + (final.finalProtocol as { externalN: number | null }).externalN = 38; + expect(() => assertFixedTraceEvaluationProtocol(final)).toThrow( + "external final is unavailable", + ); }); }); diff --git a/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts new file mode 100644 index 0000000000..76061a67cf --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; +import { + createFixedTraceEvaluatorCoordinator, + FixedTraceLedgerValidationError, + type FixedTraceActualInvocation, + type FixedTraceExpectedInvocation, +} from "../../../src/addie/eval/fixed-trace-evaluator-coordinator.js"; + +const coordinator = createFixedTraceEvaluatorCoordinator({ + hmacKey: new Uint8Array(32).fill(7), + keyId: "test-evaluator-custody-v1", +}); +const expected = ( + caseId: string, + invocation: number, +): FixedTraceExpectedInvocation => ({ + runId: "run-1", + phaseId: "stage_1_smoke", + caseId, + armId: "arm-1", + stage: "generation", + invocation, + attempt: 1, + requested: { + provider: "anthropic", + model: "claude-sonnet-5", + effort: "provider_default", + identityPolicy: "exact_model_identity_v1", + }, + controls: { + promptSha256: "a", + systemSha256: "b", + messagesSha256: "c", + toolSchemaSha256: "d", + providerRequestSha256: "e", + presentedToolNames: ["search_docs"], + presentedToolOrderSha256: "f", + simulatorReceiptProvenanceSha256: "g", + simulatorControlsSha256: "h", + architectureSha256: "i", + admissionSha256: "j", + configSha256: "k", + pricingSha256: "l", + limitsSha256: "m", + retryCacheSamplingSha256: "n", + failureDenominatorId: "all-planned-invocations-v1", + }, +}); +const actual = ( + entry: FixedTraceExpectedInvocation, +): FixedTraceActualInvocation => ({ + ...entry, + returned: { + provider: "anthropic", + model: "claude-sonnet-5", + identityPolicy: "exact_model_identity_v1", + }, + toolCallsSha256: "o", + toolInputsSha256: "p", + toolResultsSha256: "q", + startedAt: "2026-09-05T00:00:00.000Z", + finishedAt: "2026-09-05T00:00:01.000Z", + latencyMs: 1_000, + usage: { + inputTokens: 1, + outputTokens: 1, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + pricing: { profileId: "p", costUsd: 0.000001 }, + terminalStatus: "complete", + errorCode: null, +}); +const contract = () => + coordinator.issueExpectedSequence({ + runId: "run-1", + protocolFingerprint: "protocol", + manifestFingerprint: "manifest", + entries: [expected("case-a", 1), expected("case-b", 2)], + }); + +describe("fixed-trace evaluator-owned evidence coordinator", () => { + it("authenticates and validates the complete pre-dispatch sequence", () => { + const issued = contract(); + expect( + coordinator.validate(issued, issued.entries.map(actual)), + ).toMatchObject({ + complete: true, + plannedDenominator: 2, + observedDenominator: 2, + hardFailureDenominator: 0, + }); + }); + it.each([ + [ + "omission", + (issued: ReturnType) => [actual(issued.entries[1]!)], + ], + [ + "insertion", + (issued: ReturnType) => [ + { ...actual(issued.entries[0]!), caseId: "unplanned" }, + ], + ], + [ + "duplication", + (issued: ReturnType) => [ + actual(issued.entries[0]!), + actual(issued.entries[0]!), + actual(issued.entries[1]!), + ], + ], + [ + "substitution", + (issued: ReturnType) => [ + { + ...actual(issued.entries[0]!), + requested: { ...issued.entries[0]!.requested, model: "wrong-model" }, + }, + ], + ], + [ + "reordering", + (issued: ReturnType) => [ + actual(issued.entries[1]!), + actual(issued.entries[0]!), + ], + ], + ] as const)( + "rejects %s through its distinct validation branch", + (kind, build) => { + const issued = contract(); + try { + coordinator.validate(issued, build(issued)); + } catch (error) { + expect(error).toBeInstanceOf(FixedTraceLedgerValidationError); + expect((error as FixedTraceLedgerValidationError).tamperClass).toBe( + kind, + ); + return; + } + throw new Error("expected ledger validation failure"); + }, + ); + it("rejects contract restamping and halts unknown exposure", () => { + const issued = contract(); + expect(() => + coordinator.validate( + { ...issued, signature: "00".repeat(32) }, + issued.entries.map(actual), + ), + ).toThrow("authentication"); + expect(() => + coordinator.validate(issued, [ + { ...actual(issued.entries[0]!), terminalStatus: "unknown_exposure" }, + ]), + ).toThrow("unknown provider exposure"); + }); +}); diff --git a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts b/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts deleted file mode 100644 index c64e6df933..0000000000 --- a/server/tests/unit/addie/fixed-trace-experiment-plan.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { FIXED_TRACE_EXPERIMENT_PLAN_VERSION, FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION, assertFixedTraceExperimentPlan, estimateFixedTraceExperiment, fixedTraceCandidatePlanFingerprint, fixedTraceExperimentPlanFingerprint, fixedTraceTrustedManifestFingerprint, validateFixedTraceExperimentPlanOffline, validateFixedTraceRawAuditableLedgerOffline, type FixedTraceExperimentPlan } from '../../../src/addie/eval/fixed-trace-experiment-plan.js'; -import { FIXED_TRACE_PARTITION_MANIFEST, FIXED_TRACE_PARTITION_MANIFEST_SHA256, FIXED_TRACE_PARTITION_MANIFEST_VERSION } from '../../../src/addie/eval/fixed-trace-partition.js'; -import { CLAUDE_PRICING_VERSION } from '../../../src/addie/claude-pricing.js'; -import { CODE_VERSION } from '../../../src/addie/config-version.js'; -import { FIXED_TRACE_STAGE_CONTROL_VERSION, FIXED_TRACE_SUITE } from '../../../src/addie/eval/fixed-trace-suite.js'; - -const HASH = 'a'.repeat(64); -function plan(): FixedTraceExperimentPlan { - const inputBytesByTrace = Object.fromEntries(FIXED_TRACE_PARTITION_MANIFEST.development.map((id) => [id, [100]])); - return { version: FIXED_TRACE_EXPERIMENT_PLAN_VERSION, id: 'offline-v1', trustedManifestId: 'unissued', sourceId: 'fixture', sourceRevision: 'v1', pricingAsOf: '2026-09-05T12:00:00.000Z', sourceBundleSha256: HASH, gitCommit: 'a'.repeat(40), gitDirty: false, addieCodeVersion: CODE_VERSION, stageControlVersion: FIXED_TRACE_STAGE_CONTROL_VERSION, traceSuiteSha256: HASH, promptConfigVersion: HASH, toolSchemaSha256: HASH, toolDefinitionProvenance: 'fixture_local', providerDegradationInjectionEnabled: true, partition: { manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, selected: 'development' }, ordering: { seed: 'seed' }, budgets: { candidateCeilingUsd: 1, judgeCeilingUsd: 1 }, arms: [{ id: 'router-r1', architecture: 'two_stage_llm_router', screeningStage: 'router_only_screen', repetitionIndex: 1, router: { provider: 'anthropic', model: 'claude-haiku-4-5', reasoningEffort: 'provider_default', pricingVersion: CLAUDE_PRICING_VERSION, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0, samplingMode: 'provider_no_sampling_control', temperature: null, cacheMode: 'disabled', requestBounds: { inputBytesByTrace } } }] }; -} - -describe('fixed-trace experiment plan offline boundary', () => { - it('is diagnostic only and has no trust or dispatch lock', () => { - expect(validateFixedTraceExperimentPlanOffline(plan())).toMatchObject({ diagnosticOnly: true, comparisonEligible: false, dispatchable: false, trustedLock: false }); - }); - it('rejects inherited, accessor, proxy, extra-field, and unpriced Terra input', () => { - const inherited = Object.assign(Object.create(plan()), { version: FIXED_TRACE_EXPERIMENT_PLAN_VERSION }); - expect(() => validateFixedTraceExperimentPlanOffline(inherited)).toThrow('plain object'); - const getter = plan(); Object.defineProperty(getter, 'id', { enumerable: true, get: () => 'getter' }); - expect(() => validateFixedTraceExperimentPlanOffline(getter)).toThrow('own enumerable data'); - expect(() => validateFixedTraceExperimentPlanOffline(new Proxy(plan(), {}))).toThrow('Proxy'); - const extra = plan() as FixedTraceExperimentPlan & { extra: boolean }; extra.extra = true; - expect(() => validateFixedTraceExperimentPlanOffline(extra)).toThrow('unknown'); - const terra = plan(); terra.arms[0].router!.model = 'gpt-5.6-terra'; - expect(() => validateFixedTraceExperimentPlanOffline(terra)).toThrow('Unavailable immutable pricing'); - }); - it('does not lose prototype-pollution keys at plan and raw-ledger boundaries', () => { - for (const key of ['__proto__', 'prototype', 'constructor']) { - const hostile = plan() as any; - Object.defineProperty(hostile, key, { enumerable: true, value: { poisoned: true } }); - expect(() => fixedTraceExperimentPlanFingerprint(hostile, () => null)).toThrow('dangerous prototype key'); - expect(() => fixedTraceCandidatePlanFingerprint(hostile)).toThrow('dangerous prototype key'); - } - const manifest = { id: 'clean' } as any; - const hostileManifest = { id: 'clean' } as any; - Object.defineProperty(hostileManifest, '__proto__', { enumerable: true, value: { poisoned: true } }); - expect(fixedTraceTrustedManifestFingerprint(hostileManifest)) - .not.toBe(fixedTraceTrustedManifestFingerprint(manifest)); - }); - it('does not invoke a hostile getter before rejecting it, and detaches estimates', () => { - const hostile = plan() as any; - let reads = 0; - Object.defineProperty(hostile, 'id', { enumerable: true, get() { reads += 1; return 'forged'; } }); - expect(() => validateFixedTraceExperimentPlanOffline(hostile)).toThrow('own enumerable data'); - expect(reads).toBe(0); - const mutable = plan(); - const estimate = estimateFixedTraceExperiment(mutable, () => null); - mutable.arms[0].router!.maxOutputTokens = 999; - expect(estimate.candidate.reservations[0]?.outputTokens).toBe(FIXED_TRACE_PARTITION_MANIFEST.development.length * 10); - expect(Object.isFrozen(estimate.candidate.reservations)).toBe(true); - (mutable.arms as any).extra = true; - expect(() => validateFixedTraceExperimentPlanOffline(mutable)).toThrow('extra array property'); - }); - it('reclassifies the repository-visible split as development validation, never a confirmatory holdout', () => { - const validation = plan() as any; - validation.partition = { - manifestVersion: FIXED_TRACE_PARTITION_MANIFEST_VERSION, - manifestSha256: FIXED_TRACE_PARTITION_MANIFEST_SHA256, - selected: 'repository_visible_development_validation', - }; - validation.arms[0].router.requestBounds.inputBytesByTrace = Object.fromEntries( - FIXED_TRACE_PARTITION_MANIFEST.repositoryVisibleDevelopmentValidation.map((id) => [id, [100]]), - ); - expect(validateFixedTraceExperimentPlanOffline(validation)).toMatchObject({ diagnosticOnly: true, dispatchable: false }); - expect(FIXED_TRACE_REPOSITORY_VISIBLE_VALIDATION_LIMITATION).toContain('not_confirmatory_holdout'); - expect(() => assertFixedTraceExperimentPlan(validation, () => null)).toThrow('locked'); - validation.partition.selected = 'holdout'; - expect(() => validateFixedTraceExperimentPlanOffline(validation)).toThrow('Only repository-visible development partitions'); - }); - it('rejects every syntax-shaped ledger until a trusted coordinator binds exact execution expectations', () => { - const current = plan(); - const entries = FIXED_TRACE_PARTITION_MANIFEST.development.map((traceId, index) => ({ sequence: index + 1, phaseId: 'router_only_screen' as const, armId: 'router-r1', repetitionIndex: 1, traceId, stage: 'router' as const, callIndex: 1, attemptIndex: 1, dispatched: false, requestedProvider: 'anthropic' as const, requestedModel: 'claude-haiku-4-5', returnedProvider: null, returnedModel: null, promptSha256: HASH, systemSha256: HASH, docsSha256: HASH, toolSchemaSha256: HASH, providerRequestSha256: null, responseSha256: null, rawRequestArtifact: null, rawResponseArtifact: null, exactToolNames: FIXED_TRACE_SUITE.find((item) => item.id === traceId)!.toolFixtures.map((fixture) => fixture.name), caseControlSha256: HASH, executionEnvelopeSha256: HASH, directAdmissionSha256: HASH, simulatorReceiptSha256: HASH, simulatorResultProvenanceSha256: HASH, maxOutputTokens: 10, timeoutMs: 1_000, maxIterations: 1, transportRetries: 0 as const, reasoningEffort: 'provider_default' as const, samplingMode: 'provider_no_sampling_control' as const, cacheMode: 'disabled' as const, pricingProfileId: CLAUDE_PRICING_VERSION, failureDenominatorId: 'all-planned-case-stage-invocations-v1', status: 'not_dispatched' as const, finishReason: null, usage: null, estimatedCostUsd: null })); - const ledger = { version: 'addie-fixed-trace-raw-ledger-v1' as const, trustedManifestSha256: HASH, planFingerprint: validateFixedTraceExperimentPlanOffline(current).planFingerprint, budgetIdentitySha256: estimateFixedTraceExperiment(current, () => null).budgetIdentitySha256, entries }; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); - const hostileLedger = { ...ledger } as any; - Object.defineProperty(hostileLedger, '__proto__', { enumerable: true, value: { poisoned: true } }); - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, hostileLedger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); - ledger.entries[1].sequence = 1; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); - ledger.entries[1].sequence = 2; ledger.entries[0].exactToolNames = ['tampered']; - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); - ledger.entries[0].exactToolNames = FIXED_TRACE_SUITE.find((item) => item.id === ledger.entries[0].traceId)!.toolFixtures.map((fixture) => fixture.name); ledger.entries[0].returnedProvider = 'google'; ledger.entries[0].returnedModel = 'gemini-3.7-flash'; - ledger.entries[0].callIndex = 99; - ledger.entries[0].promptSha256 = 'f'.repeat(64); - ledger.entries[0].caseControlSha256 = 'e'.repeat(64); - ledger.entries[0].executionEnvelopeSha256 = 'd'.repeat(64); - ledger.entries[0].directAdmissionSha256 = 'c'.repeat(64); - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); - ledger.entries[0].returnedProvider = null; ledger.entries[0].returnedModel = null; - ledger.trustedManifestSha256 = 'b'.repeat(64); - expect(() => validateFixedTraceRawAuditableLedgerOffline(current, ledger, HASH)).toThrow('unavailable pending a trusted evaluator-owned coordinator'); - }); -}); diff --git a/server/tests/unit/addie/fixed-trace-judge.test.ts b/server/tests/unit/addie/fixed-trace-judge.test.ts index 8722cca55a..c5ac1ca374 100644 --- a/server/tests/unit/addie/fixed-trace-judge.test.ts +++ b/server/tests/unit/addie/fixed-trace-judge.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it } from "vitest"; import { FIXED_TRACE_MIN_INDEPENDENT_JUDGES, buildFixedTraceJudgeRequest, @@ -6,18 +6,18 @@ import { runIndependentFixedTraceJudges, summarizeFixedTraceJudges, type FixedTraceJudgeConfig, -} from '../../../src/addie/eval/fixed-trace-judge.js'; +} from "../../../src/addie/eval/fixed-trace-judge.js"; import { BudgetedFixedTraceProvider, FixedTraceBudget, fixedTraceResponsePricingPolicy, -} from '../../../src/addie/eval/fixed-trace-budget.js'; +} from "../../../src/addie/eval/fixed-trace-budget.js"; import { FIXED_TRACE_SUITE, FIXED_TRACE_SUITE_VERSION, type FixedTraceModelStageMetadata, type FixedTraceObservation, -} from '../../../src/addie/eval/fixed-trace-suite.js'; +} from "../../../src/addie/eval/fixed-trace-suite.js"; import type { ModelProvider, ModelProviderCapabilities, @@ -26,13 +26,13 @@ import type { ModelRespondOptions, NormalizedModelEvent, PreparedModelInvocation, -} from '../../../src/addie/model-providers/model-provider.js'; +} from "../../../src/addie/model-providers/model-provider.js"; const CAPABILITIES: ModelProviderCapabilities = { streaming: false, structuredOutput: true, reasoning: true, - reasoningEfforts: ['provider_default', 'none', 'low'], + reasoningEfforts: ["provider_default", "none", "low"], customTools: false, providerWebSearch: false, imageInput: false, @@ -40,14 +40,15 @@ const CAPABILITIES: ModelProviderCapabilities = { }; const PRICING = { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', + profileId: "openai-gpt-5.6-luna-2026-08-26", inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset' as const, - cacheWriteAccounting: 'unsupported' as const, - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', + cacheReadAccounting: "subset" as const, + cacheWriteAccounting: "unsupported" as const, + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", }; class ScriptedJudgeProvider implements ModelProvider { @@ -57,7 +58,7 @@ class ScriptedJudgeProvider implements ModelProvider { constructor( readonly id: ModelProviderId, private readonly output: string | string[], - private readonly finishReason: 'stop' | 'length' = 'stop', + private readonly finishReason: "stop" | "length" = "stop", private readonly includeProviderState = false, ) {} @@ -67,7 +68,11 @@ class ScriptedJudgeProvider implements ModelProvider { model: request.model, capabilities: this.capabilities, requestMetadata: request.requestMetadata, - providerRequest: { model: request.model, messages: request.messages, max: request.maxOutputTokens }, + providerRequest: { + model: request.model, + messages: request.messages, + max: request.maxOutputTokens, + }, }; } @@ -80,9 +85,9 @@ class ScriptedJudgeProvider implements ModelProvider { this.dispatches++; const outputs = Array.isArray(this.output) ? this.output : [this.output]; const providerState = { - type: 'provider_state' as const, + type: "provider_state" as const, provider: this.id, - kind: 'thinking', + kind: "thinking", }; const response = { provider: this.id, @@ -90,137 +95,172 @@ class ScriptedJudgeProvider implements ModelProvider { id: `${this.id}-judge-response`, content: [ ...(this.includeProviderState ? [providerState] : []), - ...outputs.map((text) => ({ type: 'text' as const, text })), + ...outputs.map((text) => ({ type: "text" as const, text })), ], finishReason: this.finishReason, providerFinishReason: this.finishReason, usage: { inputTokens: 100, outputTokens: 20 }, }; - yield { type: 'response_start', provider: this.id, model: request.model, id: response.id }; - if (this.includeProviderState) yield { type: 'provider_state', index: 0, state: providerState }; + yield { + type: "response_start", + provider: this.id, + model: request.model, + id: response.id, + }; + if (this.includeProviderState) + yield { type: "provider_state", index: 0, state: providerState }; for (const [index, text] of outputs.entries()) { - yield { type: 'text_delta', index: index + (this.includeProviderState ? 1 : 0), text }; + yield { + type: "text_delta", + index: index + (this.includeProviderState ? 1 : 0), + text, + }; } - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; } } function stage(provider: ModelProviderId): FixedTraceModelStageMetadata { return { - source: 'provider', + source: "provider", dispatched: true, requestedProvider: provider, requestedModel: `${provider}-candidate-secret-model`, returnedProvider: provider, returnedModel: `${provider}-candidate-secret-model`, - modelResolution: 'exact', - promptSha256: 'a'.repeat(64), - providerRequestSha256: 'b'.repeat(64), - reasoningEffort: 'none', + modelResolution: "exact", + promptSha256: "a".repeat(64), + providerRequestSha256: "b".repeat(64), + reasoningEffort: "none", maxOutputTokens: 300, timeoutMs: 30_000, maxIterations: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', + samplingMode: "provider_no_sampling_control", temperature: null, usageKnown: true, usage: { inputTokens: 1, outputTokens: 1 }, estimatedCostUsd: 0.001, - pricingSource: 'synthetic', + pricingSource: "synthetic", latencyMs: 10, }; } -function observation(traceId: string, provider: ModelProviderId = 'anthropic'): FixedTraceObservation { +function observation( + traceId: string, + provider: ModelProviderId = "anthropic", +): FixedTraceObservation { return { traceId, metadata: { - runId: 'candidate-secret-run-id', + runId: "candidate-secret-run-id", traceSuiteVersion: FIXED_TRACE_SUITE_VERSION, - traceSuiteSha256: 'c'.repeat(64), - sourceBundleSha256: 'd'.repeat(64), - gitCommit: '0123456789abcdef', + traceSuiteSha256: "c".repeat(64), + sourceBundleSha256: "d".repeat(64), + gitCommit: "0123456789abcdef", gitDirty: false, - addieCodeVersion: 'test', - promptConfigVersion: 'test', - toolSchemaSha256: 'e'.repeat(64), + addieCodeVersion: "test", + promptConfigVersion: "test", + toolSchemaSha256: "e".repeat(64), router: stage(provider), generation: stage(provider), }, - terminalStage: 'generation', - terminalStatus: 'complete', + terminalStage: "generation", + terminalStatus: "complete", boundaryReason: null, localReplacementReason: null, - finishReason: 'stop', - output: 'AdCP uses typed tasks between buyer and seller agents.', + finishReason: "stop", + output: "AdCP uses typed tasks between buyer and seller agents.", flagged: false, - route: { action: 'respond', toolSets: ['knowledge'] }, - tools: [{ - name: 'search_docs', - description: 'Search synthetic official documentation.', - input: { query: 'task model' }, - effect: 'read', - policyDisposition: 'allowed', - resultStatus: 'ok', - simulated: true, - }], + route: { action: "respond", toolSets: ["knowledge"] }, + tools: [ + { + name: "search_docs", + description: "Search synthetic official documentation.", + input: { query: "task model" }, + effect: "read", + policyDisposition: "allowed", + resultStatus: "ok", + simulated: true, + }, + ], }; } function config(provider: ModelProvider): FixedTraceJudgeConfig { return { provider, - model: provider.id === 'openai' ? 'gpt-5.6-luna' : `${provider.id}-judge-model`, - reasoningEffort: provider.id === 'google' ? 'low' : 'none', + model: + provider.id === "openai" ? "gpt-5.6-luna" : `${provider.id}-judge-model`, + reasoningEffort: provider.id === "google" ? "low" : "none", maxOutputTokens: 200, timeoutMs: 30_000, pricing: PRICING, }; } -describe('fixed-trace independent judge', () => { - const trace = FIXED_TRACE_SUITE.find((candidate) => candidate.id === 'knowledge-task-model')!; +describe("fixed-trace independent judge", () => { + const trace = FIXED_TRACE_SUITE.find( + (candidate) => candidate.id === "knowledge-task-model", + )!; - it('builds a blinded request without candidate model, provider, or run identity', () => { + it("builds a blinded request without candidate model, provider, or run identity", () => { const candidate = observation(trace.id); const request = buildFixedTraceJudgeRequest(trace, candidate, { - model: 'judge-model', - reasoningEffort: 'none', + model: "judge-model", + reasoningEffort: "none", maxOutputTokens: 200, }); const serialized = JSON.stringify(request); - expect(serialized).not.toContain('candidate-secret'); - expect(serialized).not.toContain('anthropic'); - expect(serialized).not.toContain('Official task lifecycle: if work is asynchronous'); - expect(serialized).toContain('candidate_answer'); - expect(serialized).toContain('Search synthetic official documentation.'); - expect(serialized).toContain('task model'); - expect(request.requestMetadata).toEqual({ purpose: 'fixed_trace_blinded_judge', trace_id: trace.id }); + expect(serialized).not.toContain("candidate-secret"); + expect(serialized).not.toContain("anthropic"); + expect(serialized).not.toContain( + "Official task lifecycle: if work is asynchronous", + ); + expect(serialized).toContain("candidate_answer"); + expect(serialized).toContain("Search synthetic official documentation."); + expect(serialized).toContain("task model"); + expect(request.requestMetadata).toEqual({ + purpose: "fixed_trace_blinded_judge", + trace_id: trace.id, + }); expect(request.outputSchema).toMatchObject({ - name: 'fixed_trace_judge_verdict', + name: "fixed_trace_judge_verdict", strict: true, schema: { - required: ['pass', 'score', 'reason', 'finding'], + required: ["pass", "score", "reason", "finding"], additionalProperties: false, }, }); }); - it('accepts a strict, internally consistent verdict with complete provenance', async () => { - const provider = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct","finding":"The answer matches the executed tool evidence."}'); - const result = await judgeFixedTraceObservation(trace, observation(trace.id), config(provider)); + it("accepts a strict, internally consistent verdict with complete provenance", async () => { + const provider = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer matches the executed tool evidence."}', + ); + const result = await judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ); expect(result).toMatchObject({ - status: 'judged', + status: "judged", failureReason: null, - verdict: { pass: true, score: 4, reason: 'correct', finding: 'The answer matches the executed tool evidence.' }, + verdict: { + pass: true, + score: 4, + reason: "correct", + finding: "The answer matches the executed tool evidence.", + }, metadata: { candidateIdentityMetadataExposed: false, - requestedProvider: 'openai', - returnedProvider: 'openai', + requestedProvider: "openai", + returnedProvider: "openai", usageKnown: true, maxIterations: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', + samplingMode: "provider_no_sampling_control", temperature: null, }, }); @@ -230,108 +270,190 @@ describe('fixed-trace independent judge', () => { expect(result.metadata.estimatedCostUsd).toBeCloseTo(0.000044); }); - it('joins a valid verdict split across provider text blocks', async () => { - const provider = new ScriptedJudgeProvider('openai', [ + it("joins a valid verdict split across provider text blocks", async () => { + const provider = new ScriptedJudgeProvider("openai", [ '{"pass":true,', '"score":3,"reason":"correct","finding":"The answer is supported."}', ]); - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(provider))) - .resolves.toMatchObject({ - status: 'judged', - verdict: { pass: true, score: 3, reason: 'correct' }, - }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ), + ).resolves.toMatchObject({ + status: "judged", + verdict: { pass: true, score: 3, reason: "correct" }, + }); }); - it('accepts a verdict accompanied only by authenticated provider thinking state', async () => { + it("accepts a verdict accompanied only by authenticated provider thinking state", async () => { const provider = new ScriptedJudgeProvider( - 'anthropic', + "anthropic", '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', - 'stop', + "stop", true, ); - await expect(judgeFixedTraceObservation(trace, observation(trace.id, 'openai'), config(provider))) - .resolves.toMatchObject({ - status: 'judged', - verdict: { pass: true, score: 4, reason: 'correct' }, - }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id, "openai"), + config(provider), + ), + ).resolves.toMatchObject({ + status: "judged", + verdict: { pass: true, score: 4, reason: "correct" }, + }); }); - it('rejects inconsistent or truncated judge output', async () => { - const inconsistent = new ScriptedJudgeProvider('openai', '{"pass":true,"score":2,"reason":"correct"}'); - const truncated = new ScriptedJudgeProvider('google', '{"pass":true', 'length'); - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(inconsistent))) - .resolves.toMatchObject({ status: 'invalid', failureReason: 'judge_output_invalid' }); - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(truncated))) - .resolves.toMatchObject({ status: 'invalid', failureReason: 'judge_output_truncated' }); + it("rejects inconsistent or truncated judge output", async () => { + const inconsistent = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":2,"reason":"correct"}', + ); + const truncated = new ScriptedJudgeProvider( + "google", + '{"pass":true', + "length", + ); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(inconsistent), + ), + ).resolves.toMatchObject({ + status: "invalid", + failureReason: "judge_output_invalid", + }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(truncated), + ), + ).resolves.toMatchObject({ + status: "invalid", + failureReason: "judge_output_truncated", + }); }); - it('requires a bounded audit finding in every verdict', async () => { + it("requires a bounded audit finding in every verdict", async () => { const missing = new ScriptedJudgeProvider( - 'openai', + "openai", '{"pass":true,"score":4,"reason":"correct"}', ); const blank = new ScriptedJudgeProvider( - 'openai', + "openai", '{"pass":true,"score":4,"reason":"correct","finding":""}', ); const oversized = new ScriptedJudgeProvider( - 'openai', - JSON.stringify({ pass: true, score: 4, reason: 'correct', finding: 'x'.repeat(241) }), + "openai", + JSON.stringify({ + pass: true, + score: 4, + reason: "correct", + finding: "x".repeat(241), + }), ); for (const provider of [missing, blank, oversized]) { - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(provider))) - .resolves.toMatchObject({ status: 'invalid', failureReason: 'judge_output_invalid' }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ), + ).resolves.toMatchObject({ + status: "invalid", + failureReason: "judge_output_invalid", + }); } }); - it('refuses a same-provider judge before dispatch', async () => { - const provider = new ScriptedJudgeProvider('anthropic', '{"pass":true,"score":4,"reason":"correct"}'); - const result = await judgeFixedTraceObservation(trace, observation(trace.id), config(provider)); - expect(result).toMatchObject({ status: 'skipped', failureReason: 'judge_not_independent' }); + it("refuses a same-provider judge before dispatch", async () => { + const provider = new ScriptedJudgeProvider( + "anthropic", + '{"pass":true,"score":4,"reason":"correct"}', + ); + const result = await judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ); + expect(result).toMatchObject({ + status: "skipped", + failureReason: "judge_not_independent", + }); expect(provider.dispatches).toBe(0); }); - it('also excludes a returned fallback provider from the judge panel', async () => { + it("also excludes a returned fallback provider from the judge panel", async () => { const candidate = observation(trace.id); - candidate.metadata.generation.returnedProvider = 'google'; - candidate.metadata.generation.returnedModel = 'google-fallback-secret-model'; - candidate.metadata.generation.modelResolution = 'provider_canonicalized'; - const provider = new ScriptedJudgeProvider('google', '{"pass":true,"score":4,"reason":"correct"}'); - const result = await judgeFixedTraceObservation(trace, candidate, config(provider)); - expect(result).toMatchObject({ status: 'skipped', failureReason: 'judge_not_independent' }); + candidate.metadata.generation.returnedProvider = "google"; + candidate.metadata.generation.returnedModel = + "google-fallback-secret-model"; + candidate.metadata.generation.modelResolution = "provider_canonicalized"; + const provider = new ScriptedJudgeProvider( + "google", + '{"pass":true,"score":4,"reason":"correct"}', + ); + const result = await judgeFixedTraceObservation( + trace, + candidate, + config(provider), + ); + expect(result).toMatchObject({ + status: "skipped", + failureReason: "judge_not_independent", + }); expect(provider.dispatches).toBe(0); }); - it('attributes a budget rejection without dispatching the judge', async () => { - const delegate = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct"}'); + it("attributes a budget rejection without dispatching the judge", async () => { + const delegate = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct"}', + ); const budget = new FixedTraceBudget(0.000001); const provider = new BudgetedFixedTraceProvider( delegate, budget, PRICING, - fixedTraceResponsePricingPolicy('openai', 'gpt-5.6-luna', PRICING), + fixedTraceResponsePricingPolicy("openai", "gpt-5.6-luna", PRICING), + ); + const result = await judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), ); - const result = await judgeFixedTraceObservation(trace, observation(trace.id), config(provider)); expect(result).toMatchObject({ - status: 'not_dispatched_budget', - failureReason: 'judge_budget_rejected', + status: "not_dispatched_budget", + failureReason: "judge_budget_rejected", metadata: { usageKnown: false, estimatedCostUsd: 0 }, }); expect(result.metadata.providerRequestSha256).toMatch(/^[a-f0-9]{64}$/); expect(delegate.dispatches).toBe(0); }); - it('requires and summarizes two distinct non-candidate judge providers', async () => { + it("requires and summarizes two distinct non-candidate judge providers", async () => { const candidate = observation(trace.id); - const openai = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}'); - const google = new ScriptedJudgeProvider('google', '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}'); + const openai = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', + ); + const google = new ScriptedJudgeProvider( + "google", + '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}', + ); const judgments = await runIndependentFixedTraceJudges( [trace], [candidate], [config(openai), config(google)], ); expect(judgments).toHaveLength(FIXED_TRACE_MIN_INDEPENDENT_JUDGES); - expect(summarizeFixedTraceJudges([trace], [candidate], judgments)).toMatchObject({ + expect( + summarizeFixedTraceJudges([trace], [candidate], judgments), + ).toMatchObject({ expectedCases: 1, expectedJudgments: 2, observedJudgments: 2, @@ -344,26 +466,39 @@ describe('fixed-trace independent judge', () => { }); }); - it('rejects an incomplete independent judge panel before any judge dispatch', async () => { - const openai = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct"}'); - await expect(runIndependentFixedTraceJudges( - [trace], - [observation(trace.id)], - [config(openai)], - )).rejects.toThrow('requires at least two independent judge providers'); + it("rejects an incomplete independent judge panel before any judge dispatch", async () => { + const openai = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct"}', + ); + await expect( + runIndependentFixedTraceJudges( + [trace], + [observation(trace.id)], + [config(openai)], + ), + ).rejects.toThrow("requires at least two independent judge providers"); expect(openai.dispatches).toBe(0); }); - it('records disagreement as a failed consensus without hiding completed coverage', async () => { + it("records disagreement as a failed consensus without hiding completed coverage", async () => { const candidate = observation(trace.id); - const openai = new ScriptedJudgeProvider('openai', '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}'); - const google = new ScriptedJudgeProvider('google', '{"pass":false,"score":2,"reason":"incomplete","finding":"The answer omits a required criterion."}'); + const openai = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}', + ); + const google = new ScriptedJudgeProvider( + "google", + '{"pass":false,"score":2,"reason":"incomplete","finding":"The answer omits a required criterion."}', + ); const judgments = await runIndependentFixedTraceJudges( [trace], [candidate], [config(openai), config(google)], ); - expect(summarizeFixedTraceJudges([trace], [candidate], judgments)).toMatchObject({ + expect( + summarizeFixedTraceJudges([trace], [candidate], judgments), + ).toMatchObject({ judgmentCoverageRate: 1, consensusPassRate: 0, disagreementRate: 1, From 988ad6e0e6ae6d1903f4175a3a8872bde93d105b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 21:31:51 +0000 Subject: [PATCH 15/16] fix(addie): fail closed planner evidence gates --- .../eval/fixed-trace-evaluation-protocol.ts | 218 +++++++++++++++--- .../eval/fixed-trace-evaluator-coordinator.ts | 69 ++++-- server/src/addie/eval/fixed-trace-judge.ts | 16 +- server/src/addie/eval/fixed-trace-runner.ts | 15 ++ .../fixed-trace-evaluation-protocol.test.ts | 96 +++++++- .../fixed-trace-evaluator-coordinator.test.ts | 49 +++- .../unit/addie/fixed-trace-judge.test.ts | 24 ++ .../unit/addie/fixed-trace-runner.test.ts | 10 + 8 files changed, 428 insertions(+), 69 deletions(-) diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index 7c4e26cfcc..79224508a2 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -35,7 +35,7 @@ import { import { FIXED_TRACE_CORPUS } from "./fixed-trace-suite.js"; export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = - "addie-fixed-trace-evaluation-protocol-v2" as const; + "addie-fixed-trace-evaluation-protocol-v3" as const; export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ version: "addie-fixed-trace-confirmatory-power-v2", @@ -44,7 +44,7 @@ export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ Object.freeze({ id: "H1-superiority", comparison: "locked-pipeline-candidate vs locked-pipeline-comparator", - endpoint: "two-judge blinded success rate", + endpoint: "two-judge blinded quality success rate", direction: "greater", marginPercentagePoints: 0, alternativeDifferencePercentagePoints: 5, @@ -52,17 +52,21 @@ export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ exactTest: "exact_conditional_mcnemar_zero_margin_only", }), Object.freeze({ - id: "H2-non-inferiority", - comparison: "locked-pipeline-candidate vs locked-pipeline-comparator", - endpoint: "two-judge blinded success rate", + id: "H2-quality-non-inferiority-for-lower-cost-pipeline", + comparison: + "lower-metered-cost locked pipeline quality vs locked-pipeline-comparator quality", + endpoint: "two-judge blinded quality success rate", direction: "not_less_than", marginPercentagePoints: -3, alternativeDifferencePercentagePoints: 0, holmOneSidedAlpha: 0.025, - exactTest: "predeclared_exact_unconditional_matched_pair_test_required", + exactTest: + "unavailable_pending_independent_Lloyd_Moldovan_score_statistic_E_plus_M_exact_unconditional_noninferiority_implementation_and_type_I_error_validation", + sensitivityOnly: + "Sidik_exact_CI_or_p_value_after_primary_Lloyd_Moldovan_verification", }), ]), - test: "exact_paired_discordance_test", + test: "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method", bootstrap: "grouped_stratified_case_level_bootstrap", exclusionRule: "hard_failures_and_missing_evidence_remain_in_denominator", repetitionsCountAsIndependentCases: false, @@ -73,6 +77,9 @@ export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ planningAlternative: "H1: +5pp over zero; H2: 0pp, three points above the -3pp NI margin", conservativeDiscordanceVarianceUpperBound: 1, + worstCaseUpperBoundsNotFinalN: true, + finalNReductionRule: + "only_sealed_never_reused_sizing_pilot_or_predeclared_blinded_arm_invariant_discordance_only_upward_internal_pilot_with_enumerated_adaptive_exact_type_I_error", externalFinalN: null, externalFinalStatus: "unavailable_pending_fingerprinted_exact_paired_discordance_power_result", @@ -89,9 +96,10 @@ export const FIXED_TRACE_CONFIRMATORY_ADMISSION = Object.freeze({ reasons: Object.freeze([ "external_final_pack_unavailable", "held_out_sizing_pilot_and_conservative_discordance_bound_unavailable", - "exact_unconditional_noninferiority_test_unavailable", + "independently_verified_Lloyd_Moldovan_E_plus_M_exact_unconditional_noninferiority_test_and_power_method_unavailable", "candidate_comparator_arm_identity_unavailable", "judge_calibration_must_be_separate_or_cross_fitted", + "privileged_evaluator_signer_and_durable_ledger_boundary_unavailable", ]), holm: Object.freeze({ K: 2, @@ -133,6 +141,7 @@ export type FixedTraceProtocolStageRole = "router" | "generation" | "judge" | "simulator"; export type FixedTraceProtocolAdmission = | "admitted_diagnostic" + | "not_admitted_common_tool_universe" | "not_admitted_architecture" | "not_evaluable_no_treatment_contrast" | "not_admitted_external_final" @@ -259,14 +268,18 @@ export const FIXED_TRACE_ADMITTED_CELLS: readonly FixedTraceAdmittedCell[] = "generation", "anthropic", "claude-haiku-4-5", - ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts, + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts.filter( + (effort) => effort === "provider_default", + ), "ANTHROPIC_PROVIDER_CAPABILITIES", ), ...cells( "generation", "anthropic", "claude-sonnet-5", - ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts, + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts.filter( + (effort) => effort === "provider_default", + ), "ANTHROPIC_PROVIDER_CAPABILITIES", ), ...cells( @@ -299,8 +312,67 @@ export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ }), ]); +/** + * This is the first *potentially* paid activity after credential-free + * admission. It is component-only: no semantic judges, no pipeline or + * architecture comparison, no execution authorization. + */ +export const FIXED_TRACE_COMPONENT_SMOKE_PLAN = Object.freeze({ + status: "not_admitted_pending_credential_free_admission", + cases: 8, + repetitions: 1, + routerCells: 10, + generationCells: 11, + totalComponentCells: 21, + maxRouterInvocationsPerCase: 1, + maxGenerationInvocationsPerCase: 2, + llmJudging: "none", + architectureClaim: "none", + providerCeilingUsd: 5, + authorization: "none", +}); + +/** Planning-only cardinalities; this does not schedule confirmation. */ +export const FIXED_TRACE_ARCHITECTURE_CELL_TRUTH = Object.freeze({ + routerCells: 10, + generationCells: 11, + directCombinations: 11, + twoStageCombinations: 110, + hybridCombinations: 110, + totalArchitectureCombinations: 231, + potentiallyLlmJudgeableProviderMatchedCombinations: 97, + mixedProviderCombinationsRequiringHumanOrFourthProvider: 134, +}); + +/** USD is operational evidence, never a percentage-point quality hypothesis. */ +export const FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE = Object.freeze({ + status: "not_admitted_pending_complete_trusted_usage_and_predeclared_economic_margin", + endpoint: "paired_metered_USD_cost_and_latency_reliability", + requiredEvidence: + "complete_trusted_usage_pricing_cost_for_every_dispatched_and_failed_timeout_unknown_exposure_invocation", + qualityHypothesisRelationship: + "separate_from_H2_quality_noninferiority", + binaryPercentagePointHypothesis: false, +}); + /** No judge can score a finalist until this evaluator-custodied record exists. */ export const FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS = Object.freeze([ + Object.freeze({ + provider: "anthropic" as const, + model: "claude-haiku-4-5", + effort: "provider_default" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), Object.freeze({ provider: "openai" as const, model: OPENAI_ROUTER_MODEL, @@ -345,6 +417,11 @@ export interface FixedTraceProtocolStage { readonly retries: 0; readonly cacheMode: "disabled"; readonly sampling: "provider_no_sampling_control"; + /** No post-terminal invocation is expected; every eligible omission is a failure. */ + readonly invocationLifecycle: + | "always_eligible; dispatched_completed_terminal_usage_cost_recorded" + | "eligible_while_prior_tool_loop_is_nonterminal; post_terminal_not_eligible; eligible_omission_is_failure" + | "eligible_after_complete_candidate; candidate_hard_failure_remains_denominator"; } export interface FixedTraceProtocolArm { readonly id: string; @@ -388,14 +465,17 @@ export interface FixedTraceEvaluationProtocol { readonly finalProtocol: { readonly status: "unavailable"; readonly familywiseAlpha: 0.025; - readonly hypothesisIds: readonly ["H1-superiority", "H2-non-inferiority"]; - readonly endpoint: "two-judge blinded success rate"; + readonly hypothesisIds: readonly [ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]; + readonly endpoint: "two-judge blinded quality success rate"; readonly externalPackDigest: null; readonly externalN: null; readonly candidatePipelineId: null; readonly comparatorPipelineId: null; readonly architectureArmId: null; - readonly pairedTest: "exact_paired_discordance_test"; + readonly pairedTest: "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method"; readonly bootstrap: "grouped_stratified_case_level_bootstrap"; readonly exclusions: "hard_failures_and_missing_evidence_remain_in_denominator"; readonly fingerprint: null; @@ -420,6 +500,12 @@ const stage = ( retries: 0, cacheMode: "disabled", sampling: "provider_no_sampling_control", + invocationLifecycle: + role === "generation" && maxInvocationsPerCase > 1 + ? "eligible_while_prior_tool_loop_is_nonterminal; post_terminal_not_eligible; eligible_omission_is_failure" + : role === "judge" + ? "eligible_after_complete_candidate; candidate_hard_failure_remains_denominator" + : "always_eligible; dispatched_completed_terminal_usage_cost_recorded", }); const routerCell = FIXED_TRACE_ADMITTED_CELLS.find( (cell) => cell.id === "router:anthropic:claude-haiku-4-5:provider_default", @@ -542,15 +628,19 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto familywiseAlpha: 0.025, hypothesisIds: Object.freeze([ "H1-superiority", - "H2-non-inferiority", - ]) as readonly ["H1-superiority", "H2-non-inferiority"], - endpoint: "two-judge blinded success rate", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]) as readonly [ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ], + endpoint: "two-judge blinded quality success rate", externalPackDigest: null, externalN: null, candidatePipelineId: null, comparatorPipelineId: null, architectureArmId: null, - pairedTest: "exact_paired_discordance_test", + pairedTest: + "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method", bootstrap: "grouped_stratified_case_level_bootstrap", exclusions: "hard_failures_and_missing_evidence_remain_in_denominator", fingerprint: null, @@ -577,7 +667,7 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto stage( cell.role, cell.id, - cell.role === "router" ? 1 : 12, + cell.role === "router" ? 1 : 2, cell.role === "router" ? 4_096 : 16_384, cell.role === "router" ? 300 : 900, ), @@ -633,7 +723,7 @@ export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProto candidate( "routed-locked-finalist", "two_stage_llm_router", - "admitted_diagnostic", + "not_admitted_common_tool_universe", [ stage("router", routerCell.id, 1, 4_096, 300), stage("generation", generatorCell.id, 12, 16_384, 900), @@ -728,6 +818,7 @@ export interface FixedTraceProtocolEstimate { failedTimeoutUnknownExposureCeilingUsd: number; contingencyUsd: number; totalCeilingUsd: number; + componentSmokeCeilingUsd: number; hybridWorstCaseRouterCalls: 138; hybridWorstCaseRouterCeilingUsd: number; armCallAccounting: readonly FixedTraceArchitectureArmCallAccounting[]; @@ -757,11 +848,23 @@ export interface FixedTraceScreeningResult { /** Pure, predeclared elimination/halving rule; repetitions estimate stability only. */ export function selectFixedTraceScreeningSurvivors( results: readonly FixedTraceScreeningResult[], + requiredCellIds: readonly string[] = FIXED_TRACE_ADMITTED_CELLS.map( + (cell) => cell.id, + ), ): readonly string[] { + const required = new Set(requiredCellIds); + if ( + required.size !== requiredCellIds.length || + required.size === 0 || + [...required].some( + (cellId) => !FIXED_TRACE_ADMITTED_CELLS.some((cell) => cell.id === cellId), + ) + ) + throw new Error("screening required cell set is invalid"); const seen = new Set(); for (const result of results) { if ( - !FIXED_TRACE_ADMITTED_CELLS.some((cell) => cell.id === result.cellId) || + !required.has(result.cellId) || seen.has(result.cellId) ) throw new Error("screening result has an unknown or duplicate cell"); @@ -779,6 +882,8 @@ export function selectFixedTraceScreeningSurvivors( throw new Error("screening result has invalid metrics"); seen.add(result.cellId); } + if (seen.size !== required.size) + throw new Error("screening requires exactly one result for every supported executable cell"); const eligible = results.filter( (result) => result.safetyFailures === 0 && @@ -814,6 +919,13 @@ export function assertFixedTraceEvaluationProtocol( protocol: FixedTraceEvaluationProtocol, ): void { assertFixedTracePartitionManifest(); + if ( + FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "router").length !== + FIXED_TRACE_ARCHITECTURE_CELL_TRUTH.routerCells || + FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "generation") + .length !== FIXED_TRACE_ARCHITECTURE_CELL_TRUTH.generationCells + ) + throw new Error("executable router/generator cell inventory differs from pinned planning truth"); if ( protocol.version !== FIXED_TRACE_EVALUATION_PROTOCOL_VERSION || protocol.baseCapabilityUniverse !== @@ -874,7 +986,32 @@ export function assertFixedTraceEvaluationProtocol( throw new Error( `phase ${phase.id} does not use corpus-derived case counts`, ); + if (phase.id === "stage_1_smoke") { + if ( + phase.uniqueCases !== FIXED_TRACE_COMPONENT_SMOKE_PLAN.cases || + phase.repetitions !== 1 || + phase.arms.length !== FIXED_TRACE_COMPONENT_SMOKE_PLAN.totalComponentCells || + phase.arms.some( + (arm) => + arm.architecture !== "none" || + arm.stages.length !== 1 || + arm.stages[0]!.role === "judge" || + (arm.stages[0]!.role === "generation" && + arm.stages[0]!.maxInvocationsPerCase !== + FIXED_TRACE_COMPONENT_SMOKE_PLAN.maxGenerationInvocationsPerCase), + ) + ) + throw new Error("stage_1 is only the pinned component-only smoke"); + } for (const arm of phase.arms) { + if ( + phase.selectionUse === "architecture_selection" && + arm.architecture === "two_stage_llm_router" && + arm.admission !== "not_admitted_common_tool_universe" + ) + throw new Error( + "architecture comparison remains not admitted without a common authenticated tool universe", + ); if ( arm.architecture === "direct_generation" && arm.admission !== "not_admitted_architecture" @@ -906,7 +1043,7 @@ export function assertFixedTraceEvaluationProtocol( "stage references an unadmitted provider/model/effort cell", ); const judges = arm.stages.filter((item) => item.role === "judge"); - if (judges.length) { + if (judges.length && arm.admission === "admitted_diagnostic") { const expectedJudges = assertPromotionGradeDualJudgeFeasibility( arm, ).map((cell) => cell.id); @@ -922,6 +1059,10 @@ export function assertFixedTraceEvaluationProtocol( } } } + // This exported plan is a pinned declaration, not a caller-editable schema. + // Validate every nested field before it can be fingerprinted or budgeted. + if (sha256(protocol) !== sha256(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL)) + throw new Error("fixed-trace protocol differs from the pinned declaration"); } export function estimateFixedTraceEvaluationProtocol( protocol: FixedTraceEvaluationProtocol = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, @@ -972,6 +1113,11 @@ export function estimateFixedTraceEvaluationProtocol( const judgeCeilingUsd = stages .filter((item) => item.role === "judge") .reduce((total, item) => total + item.ceilingUsd, 0); + const componentSmokeCeilingUsd = stages + .filter((item) => item.phaseId === "stage_1_smoke") + .reduce((total, item) => total + item.ceilingUsd, 0); + if (componentSmokeCeilingUsd > FIXED_TRACE_COMPONENT_SMOKE_PLAN.providerCeilingUsd) + throw new Error("component-only smoke exceeds its non-authorizing $5 provider ceiling"); const failedTimeoutUnknownExposureCeilingUsd = candidateCeilingUsd + judgeCeilingUsd; const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * 0.1; @@ -1025,8 +1171,9 @@ export function estimateFixedTraceEvaluationProtocol( armId: arm.id, admission: arm.admission, evaluable: - arm.architecture !== "deterministic_policy_llm_fallback_hybrid" || - developmentContrast.evaluable, + arm.admission === "admitted_diagnostic" && + (arm.architecture !== "deterministic_policy_llm_fallback_hybrid" || + developmentContrast.evaluable), localTerminalCases, routedCases, routerCalls, @@ -1045,6 +1192,7 @@ export function estimateFixedTraceEvaluationProtocol( failedTimeoutUnknownExposureCeilingUsd, contingencyUsd, totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd + contingencyUsd, + componentSmokeCeilingUsd, hybridWorstCaseRouterCalls: 138, hybridWorstCaseRouterCeilingUsd: hybridRouter.ceilingUsd, armCallAccounting: Object.freeze(armCallAccounting), @@ -1061,7 +1209,7 @@ export function providerExcludingCalibratedJudges( "promotion-grade dual-LLM judging requires a single-provider complete pipeline; mixed finalists require a human-primary path or fourth calibrated provider", ); } - return Object.freeze( + const selected = (["anthropic", "openai", "google"] as const) .filter((provider) => !candidateProviders.has(provider)) .map((provider) => @@ -1072,9 +1220,25 @@ export function providerExcludingCalibratedJudges( (provider === "openai" ? cell.effort === "none" : cell.effort === "provider_default"), - )!, - ), - ); + ), + ); + if ( + selected.length !== 2 || + selected.some((cell) => !cell) || + selected.some((cell) => { + const calibration = FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS.find( + (entry) => + entry.provider === cell!.provider && + entry.model === cell!.model && + entry.effort === cell!.effort, + ); + return !calibration || calibration.authenticatedAdmission === null; + }) + ) + throw new Error( + "no admissible provider-excluding judge pair without calibrated custodied artifacts", + ); + return Object.freeze(selected as FixedTraceAdmittedCell[]); } export function semanticJudgeCandidateProviders( arm: Pick, diff --git a/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts index 6898f77cbd..73a2c60c18 100644 --- a/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts +++ b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts @@ -5,10 +5,10 @@ import type { } from "../model-providers/model-provider.js"; /** - * Evaluator-owned custody for offline fixed-trace evidence. This module has no - * provider client and no production-handler import. Its signing key is passed - * only by the evaluator's protected configuration; candidate plans, artifacts, - * and callbacks cannot mint, alter, or restamp an expected sequence. + * Diagnostic integrity only. An importer supplies this module's key, so this + * cannot establish evaluator custody or confirmatory evidence. A separately + * injected opaque privileged signer and durable dispatcher/ledger boundary is + * required before any admission can be made. */ export const FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION = "addie-fixed-trace-evaluator-coordinator-v1" as const; @@ -101,6 +101,7 @@ export interface FixedTraceActualInvocation extends FixedTraceExpectedInvocation } export interface FixedTraceExpectedSequenceContract { readonly version: typeof FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION; + readonly keyId: string; readonly runId: string; readonly protocolFingerprint: string; readonly manifestFingerprint: string; @@ -108,6 +109,7 @@ export interface FixedTraceExpectedSequenceContract { readonly signature: string; } export interface FixedTraceEvidenceLedger { + readonly admission: "not_admitted_diagnostic_hmac_without_privileged_durable_authority"; readonly contract: FixedTraceExpectedSequenceContract; readonly entries: readonly FixedTraceActualInvocation[]; readonly complete: boolean; @@ -115,8 +117,12 @@ export interface FixedTraceEvidenceLedger { readonly plannedDenominator: number; readonly observedDenominator: number; readonly hardFailureDenominator: number; + readonly signature: string; } +const DIAGNOSTIC_ADMISSION = + "not_admitted_diagnostic_hmac_without_privileged_durable_authority" as const; + function canonical(value: unknown): string { if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); @@ -135,6 +141,15 @@ function canonical(value: unknown): string { } throw new Error("non-JSON evaluator ledger value"); } +function deepSnapshot(value: T): T { + const copy = structuredClone(value); + const freeze = (nested: unknown): unknown => { + if (nested === null || typeof nested !== "object" || Object.isFrozen(nested)) return nested; + for (const child of Object.values(nested as Record)) freeze(child); + return Object.freeze(nested); + }; + return freeze(copy) as T; +} const invocationKey = ( entry: Pick< FixedTraceExpectedInvocation, @@ -159,6 +174,8 @@ const invocationKey = ( const contractProjection = ( contract: Omit, ) => canonical(contract); +const ledgerProjection = (ledger: Omit) => + canonical(ledger); const sameExpected = ( actual: FixedTraceActualInvocation, expected: FixedTraceExpectedInvocation, @@ -210,6 +227,7 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { const verify = (contract: FixedTraceExpectedSequenceContract) => { const projection = contractProjection({ version: contract.version, + keyId: contract.keyId, runId: contract.runId, protocolFingerprint: contract.protocolFingerprint, manifestFingerprint: contract.manifestFingerprint, @@ -219,6 +237,7 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { const supplied = Buffer.from(contract.signature, "hex"); if ( contract.version !== FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION || + contract.keyId !== evaluatorConfig.keyId || expected.length !== supplied.length || !timingSafeEqual(expected, supplied) ) @@ -228,8 +247,12 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { ); }; return Object.freeze({ + admission: DIAGNOSTIC_ADMISSION, issueExpectedSequence( - input: Omit, + input: Omit< + FixedTraceExpectedSequenceContract, + "version" | "keyId" | "signature" + >, ): FixedTraceExpectedSequenceContract { if ( !input.runId.trim() || @@ -249,13 +272,14 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { ); keys.add(key); } - const unsigned = { + const unsigned = deepSnapshot({ version: FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION, + keyId: evaluatorConfig.keyId, ...input, - } as const; - return Object.freeze({ + entries: deepSnapshot(input.entries), + } as const); + return deepSnapshot({ ...unsigned, - entries: Object.freeze([...input.entries]), signature: sign(contractProjection(unsigned)), }); }, @@ -263,19 +287,21 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { contract: FixedTraceExpectedSequenceContract, actualEntries: readonly FixedTraceActualInvocation[], ): FixedTraceEvidenceLedger { - verify(contract); + const trustedContract = deepSnapshot(contract); + const trustedActualEntries = deepSnapshot(actualEntries); + verify(trustedContract); const observed: FixedTraceActualInvocation[] = []; const seen = new Set(); let halted = false; - for (const actual of actualEntries) { + for (const actual of trustedActualEntries) { if (halted) throw new FixedTraceLedgerValidationError( "unknown_exposure", "run was halted after unknown exposure", ); const key = invocationKey(actual); - const expected = contract.entries[observed.length]; - const knownIndex = contract.entries.findIndex( + const expected = trustedContract.entries[observed.length]; + const knownIndex = trustedContract.entries.findIndex( (entry) => invocationKey(entry) === key, ); if (seen.has(key)) @@ -295,7 +321,7 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { ); if (knownIndex > observed.length) { const expectedKey = invocationKey(expected); - const appearsLater = actualEntries + const appearsLater = trustedActualEntries .slice(observed.length + 1) .some((entry) => invocationKey(entry) === expectedKey); throw new FixedTraceLedgerValidationError( @@ -364,7 +390,7 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { seen.add(key); observed.push(actual); } - if (observed.length !== contract.entries.length) + if (observed.length !== trustedContract.entries.length) throw new FixedTraceLedgerValidationError( "omission", "ledger ended before its planned denominator", @@ -372,15 +398,20 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { const hardFailureDenominator = observed.filter( (entry) => entry.terminalStatus !== "complete", ).length; - return Object.freeze({ - contract, - entries: Object.freeze(observed), + const unsignedLedger = deepSnapshot({ + admission: DIAGNOSTIC_ADMISSION, + contract: trustedContract, + entries: observed, complete: true, halted: false, - plannedDenominator: contract.entries.length, + plannedDenominator: trustedContract.entries.length, observedDenominator: observed.length, hardFailureDenominator, }); + return deepSnapshot({ + ...unsignedLedger, + signature: sign(ledgerProjection(unsignedLedger)), + }); }, }); } diff --git a/server/src/addie/eval/fixed-trace-judge.ts b/server/src/addie/eval/fixed-trace-judge.ts index 294379ca54..cf2fb88fab 100644 --- a/server/src/addie/eval/fixed-trace-judge.ts +++ b/server/src/addie/eval/fixed-trace-judge.ts @@ -332,17 +332,15 @@ function validateConfig(config: FixedTraceJudgeConfig): void { } function candidateProviders(observation: FixedTraceObservation): ReadonlySet { - const generation = [ + // A router selects the candidate-visible surface, so both stages contribute + // to the pipeline under semantic review—even when generation metadata exists + // or a fallback changed the returned provider. + return new Set([ + observation.metadata.router.requestedProvider, + observation.metadata.router.returnedProvider, observation.metadata.generation.requestedProvider, observation.metadata.generation.returnedProvider, - ].filter((provider): provider is ModelProviderId => provider !== null); - const providers = generation.length > 0 - ? generation - : [ - observation.metadata.router.requestedProvider, - observation.metadata.router.returnedProvider, - ].filter((provider): provider is ModelProviderId => provider !== null); - return new Set(providers); + ].filter((provider): provider is ModelProviderId => provider !== null)); } export async function judgeFixedTraceObservation( diff --git a/server/src/addie/eval/fixed-trace-runner.ts b/server/src/addie/eval/fixed-trace-runner.ts index 1cbbdf5839..632f903b7d 100644 --- a/server/src/addie/eval/fixed-trace-runner.ts +++ b/server/src/addie/eval/fixed-trace-runner.ts @@ -592,6 +592,21 @@ function resolveTraceDefinitions( }); } +/** + * Routed replay currently obtains the presented surface from case fixtures. + * That is useful for deterministic component replay, but it is not neutral + * common-universe provenance and therefore cannot support architecture + * comparison. Keep this refusal separate from replay so it cannot be mistaken + * for an admission merely because execution succeeds. + */ +export function assertFixedTraceArchitectureComparisonPrerequisite( + config: Pick, +): never { + if (fixedTraceArchitectureArm(config.architectureArm).id === "direct_generation") + throw new Error("direct architecture comparison is not admitted without signed capture/source/thread/request binding"); + throw new Error("architecture comparison is not admitted: common authenticated base registry/schema/receipt tool universe is unavailable; fixture-local tool definitions are replay-only"); +} + export function fixedTraceToolSchemaSha256( traceSuite: readonly FixedTraceCase[], definitions: readonly AddieTool[], diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index c95b931e44..51a93a8004 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -1,10 +1,13 @@ import { describe, expect, it } from "vitest"; import { FIXED_TRACE_ADMITTED_CELLS, + FIXED_TRACE_ARCHITECTURE_CELL_TRUTH, + FIXED_TRACE_COMPONENT_SMOKE_PLAN, FIXED_TRACE_CONFIRMATORY_POWER_GATE, FIXED_TRACE_CONFIRMATORY_ADMISSION, FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT, FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS, + FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_PROTOCOL_PRICING, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, @@ -34,6 +37,8 @@ describe("fixed-trace staged protocol", () => { ).toBe(82); }); it("screens every reviewed adapter-supported provider/model/effort cell before adaptive pruning", () => { + expect(FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "router")).toHaveLength(10); + expect(FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "generation")).toHaveLength(11); for (const role of ["router", "generation"] as const) for (const provider of ["anthropic", "openai", "google"] as const) expect( @@ -60,6 +65,26 @@ describe("fixed-trace staged protocol", () => { repeats: "stability_only_not_new_cases", }, ); + expect(FIXED_TRACE_COMPONENT_SMOKE_PLAN).toMatchObject({ + status: "not_admitted_pending_credential_free_admission", + totalComponentCells: 21, + cases: 8, + repetitions: 1, + maxGenerationInvocationsPerCase: 2, + providerCeilingUsd: 5, + llmJudging: "none", + architectureClaim: "none", + }); + expect(FIXED_TRACE_ARCHITECTURE_CELL_TRUTH).toEqual({ + routerCells: 10, + generationCells: 11, + directCombinations: 11, + twoStageCombinations: 110, + hybridCombinations: 110, + totalArchitectureCombinations: 231, + potentiallyLlmJudgeableProviderMatchedCombinations: 97, + mixedProviderCombinationsRequiringHumanOrFourthProvider: 134, + }); }); it("keeps hybrid router fallback and direct admission explicit in worst-case accounting", () => { const estimate = estimateFixedTraceEvaluationProtocol(); @@ -72,6 +97,11 @@ describe("fixed-trace staged protocol", () => { architecture.arms.find((arm) => arm.architecture === "direct_generation") ?.admission, ).toBe("not_admitted_architecture"); + expect( + estimate.armCallAccounting.find( + (arm) => arm.armId === "direct-locked-finalist", + ), + ).toMatchObject({ evaluable: false, routerCalls: 0, generationCalls: 0 }); const hybrid = architecture.arms.find( (arm) => arm.architecture === "deterministic_policy_llm_fallback_hybrid", )!; @@ -131,10 +161,14 @@ describe("fixed-trace staged protocol", () => { FIXED_TRACE_CONFIRMATORY_POWER_GATE.hypotheses.map( (hypothesis) => hypothesis.id, ), - ).toEqual(["H1-superiority", "H2-non-inferiority"]); + ).toEqual([ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]); expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE).toMatchObject({ targetPower: 0.8, conservativeDiscordanceVarianceUpperBound: 1, + worstCaseUpperBoundsNotFinalN: true, hypotheses: [ { id: "H1-superiority", @@ -142,10 +176,10 @@ describe("fixed-trace staged protocol", () => { exactTest: "exact_conditional_mcnemar_zero_margin_only", }, { - id: "H2-non-inferiority", + id: "H2-quality-non-inferiority-for-lower-cost-pipeline", marginPercentagePoints: -3, exactTest: - "predeclared_exact_unconditional_matched_pair_test_required", + "unavailable_pending_independent_Lloyd_Moldovan_score_statistic_E_plus_M_exact_unconditional_noninferiority_implementation_and_type_I_error_validation", }, ], }); @@ -165,14 +199,21 @@ describe("fixed-trace staged protocol", () => { }, finalProtocolFingerprint: null, }); + expect(FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE).toMatchObject({ + status: + "not_admitted_pending_complete_trusted_usage_and_predeclared_economic_margin", + endpoint: "paired_metered_USD_cost_and_latency_reliability", + qualityHypothesisRelationship: "separate_from_H2_quality_noninferiority", + binaryPercentagePointHypothesis: false, + }); }); - it("uses two provider-excluding calibrated judge families for every candidate provider", () => { + it("fails closed until every provider-excluding judge has a custodied calibration", () => { for (const provider of ["anthropic", "openai", "google"] as const) { - const judges = providerExcludingCalibratedJudges([provider]); - expect(judges).toHaveLength(2); - expect(judges.some((judge) => judge.provider === provider)).toBe(false); + expect(() => providerExcludingCalibratedJudges([provider])).toThrow( + "no admissible provider-excluding judge pair", + ); } - expect(FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS).toHaveLength(2); + expect(FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS).toHaveLength(3); expect( FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS.every( (judge) => @@ -225,7 +266,7 @@ describe("fixed-trace staged protocol", () => { (routed.stages[0] as { cellId: string }).cellId = "router:openai:gpt-5.6-luna:none"; expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow( - "single-provider complete pipeline", + "pinned declaration", ); }); it("applies hard elimination and successive halving deterministically", () => { @@ -241,15 +282,48 @@ describe("fixed-trace staged protocol", () => { costUsd: index, }), ); - expect(selectFixedTraceScreeningSurvivors(results)).toEqual([ + const required = results.map((result) => result.cellId); + expect(selectFixedTraceScreeningSurvivors(results, required)).toEqual([ results[0]!.cellId, results[1]!.cellId, ]); - expect(selectFixedTraceScreeningSurvivors([...results].reverse())).toEqual([ + expect(selectFixedTraceScreeningSurvivors([...results].reverse(), required)).toEqual([ results[0]!.cellId, results[1]!.cellId, ]); }); + it("rejects partial and hostile screening result sets", () => { + const result = { + cellId: FIXED_TRACE_ADMITTED_CELLS[0]!.id, + safetyFailures: 0, identityFailures: 0, malformedFailures: 0, + toolLoopFailures: 0, reliabilityFailures: 0, latencyMs: 1, costUsd: 1, + }; + expect(() => selectFixedTraceScreeningSurvivors([result])).toThrow( + "exactly one result for every supported executable cell", + ); + expect(() => selectFixedTraceScreeningSurvivors( + [{ ...result, latencyMs: Number.NaN }], [result.cellId], + )).toThrow("invalid metrics"); + expect(() => selectFixedTraceScreeningSurvivors( + [result, result], [result.cellId], + )).toThrow("unknown or duplicate"); + expect(() => selectFixedTraceScreeningSurvivors( + [result], ["unknown"], + )).toThrow("required cell set is invalid"); + }); + it.each([ + (protocol: any) => { protocol.finalProtocol.familywiseAlpha = 0.5; }, + (protocol: any) => { protocol.finalProtocol.hypothesisIds[0] = "rewritten"; }, + (protocol: any) => { protocol.finalProtocol.pairedTest = "rewritten"; }, + (protocol: any) => { protocol.finalProtocol.exclusions = "drop_failures"; }, + (protocol: any) => { protocol.adaptiveRule.repeats = "inflate_N"; }, + (protocol: any) => { protocol.finalProtocol.powerResult = { admitted: true }; }, + (protocol: any) => { protocol.phases[4].arms[0].stages[0].maxOutputTokens = 1; }, + ])("rejects hostile nested protocol rewrites before fingerprinting", (mutate) => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + mutate(protocol); + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow(); + }); it("uses canonical Luna subset-cache pricing and leaves Terra/Sol inert", () => { const luna = FIXED_TRACE_PROTOCOL_PRICING.find( (profile) => profile.provider === "openai", diff --git a/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts index 76061a67cf..f95d4fa8d7 100644 --- a/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts @@ -82,14 +82,53 @@ const contract = () => describe("fixed-trace evaluator-owned evidence coordinator", () => { it("authenticates and validates the complete pre-dispatch sequence", () => { const issued = contract(); - expect( - coordinator.validate(issued, issued.entries.map(actual)), - ).toMatchObject({ + const ledger = coordinator.validate(issued, issued.entries.map(actual)); + expect(ledger).toMatchObject({ complete: true, + admission: + "not_admitted_diagnostic_hmac_without_privileged_durable_authority", plannedDenominator: 2, observedDenominator: 2, hardFailureDenominator: 0, }); + expect(issued.keyId).toBe("test-evaluator-custody-v1"); + expect(ledger.signature).toMatch(/^[a-f0-9]{64}$/); + }); + it("snapshots and freezes nested contracts and ledgers", () => { + const entries = [expected("case-a", 1), expected("case-b", 2)]; + const issued = coordinator.issueExpectedSequence({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", entries, + }); + (entries[0]!.controls.presentedToolNames as unknown as string[])[0] = "rewritten"; + expect(issued.entries[0]!.controls.presentedToolNames[0]).toBe("search_docs"); + const supplied = issued.entries.map((entry) => ({ + ...actual(entry), + controls: { + ...entry.controls, + presentedToolNames: [...entry.controls.presentedToolNames], + }, + })); + const ledger = coordinator.validate(issued, supplied); + (supplied[0]!.controls.presentedToolNames as unknown as string[])[0] = "rewritten-again"; + expect(ledger.entries[0]!.controls.presentedToolNames[0]).toBe("search_docs"); + expect(Object.isFrozen(ledger.entries[0]!.controls.presentedToolNames)).toBe(true); + expect(() => { + (ledger.entries[0]!.controls.presentedToolNames as unknown as string[])[0] = "tamper"; + }).toThrow(); + }); + it("never represents caller-keyed HMAC output as privileged custody", () => { + const arbitraryImporter = createFixedTraceEvaluatorCoordinator({ + hmacKey: new Uint8Array(32).fill(9), keyId: "arbitrary-importer-key", + }); + expect(arbitraryImporter.admission).toBe( + "not_admitted_diagnostic_hmac_without_privileged_durable_authority", + ); + const issued = arbitraryImporter.issueExpectedSequence({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", + entries: [expected("case-a", 1)], + }); + expect(arbitraryImporter.validate(issued, [actual(issued.entries[0]!)]).admission) + .toBe("not_admitted_diagnostic_hmac_without_privileged_durable_authority"); }); it.each([ [ @@ -150,6 +189,10 @@ describe("fixed-trace evaluator-owned evidence coordinator", () => { issued.entries.map(actual), ), ).toThrow("authentication"); + expect(() => coordinator.validate( + { ...issued, keyId: "wrong-custody-key" }, + issued.entries.map(actual), + )).toThrow("authentication"); expect(() => coordinator.validate(issued, [ { ...actual(issued.entries[0]!), terminalStatus: "unknown_exposure" }, diff --git a/server/tests/unit/addie/fixed-trace-judge.test.ts b/server/tests/unit/addie/fixed-trace-judge.test.ts index c5ac1ca374..40fe7c3821 100644 --- a/server/tests/unit/addie/fixed-trace-judge.test.ts +++ b/server/tests/unit/addie/fixed-trace-judge.test.ts @@ -409,6 +409,30 @@ describe("fixed-trace independent judge", () => { expect(provider.dispatches).toBe(0); }); + it("unions requested and returned router and generator providers for pipeline exclusion", async () => { + const candidate = observation(trace.id, "anthropic"); + candidate.metadata.router.returnedProvider = "openai"; + candidate.metadata.router.returnedModel = "openai-router-fallback"; + candidate.metadata.generation.requestedProvider = "google"; + candidate.metadata.generation.returnedProvider = "google"; + candidate.metadata.generation.returnedModel = "google-generator-fallback"; + const onlyRemainingProvider = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', + ); + const sameRouterProvider = new ScriptedJudgeProvider( + "anthropic", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', + ); + await expect(judgeFixedTraceObservation(trace, candidate, config(sameRouterProvider))) + .resolves.toMatchObject({ status: "skipped", failureReason: "judge_not_independent" }); + await expect(runIndependentFixedTraceJudges( + [trace], [candidate], [config(onlyRemainingProvider)], + )).rejects.toThrow("requires at least two independent judge providers"); + expect(sameRouterProvider.dispatches).toBe(0); + expect(onlyRemainingProvider.dispatches).toBe(0); + }); + it("attributes a budget rejection without dispatching the judge", async () => { const delegate = new ScriptedJudgeProvider( "openai", diff --git a/server/tests/unit/addie/fixed-trace-runner.test.ts b/server/tests/unit/addie/fixed-trace-runner.test.ts index e56dfb629d..9d9144de6f 100644 --- a/server/tests/unit/addie/fixed-trace-runner.test.ts +++ b/server/tests/unit/addie/fixed-trace-runner.test.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import Ajv from 'ajv'; import { describe, expect, it, vi } from 'vitest'; import { + assertFixedTraceArchitectureComparisonPrerequisite, buildFixedTraceGenerationRequest, fixedTraceArchitectureConfigSha256, fixedTraceToolSchemaSha256, @@ -400,6 +401,15 @@ function expandedFixtureTrace(id = 'expanded-fixture-tool'): FixedTraceCase { } describe('fixed trace artifact runner', () => { + it('fails closed architecture comparison while candidate-visible tools are fixture-local', () => { + const router = new ScriptedProvider([]); + const generation = new ScriptedProvider([]); + expect(() => assertFixedTraceArchitectureComparisonPrerequisite(config(router, generation))).toThrow( + 'common authenticated base registry/schema/receipt tool universe is unavailable', + ); + expect(router.respondCalls).toHaveLength(0); + expect(generation.respondCalls).toHaveLength(0); + }); it('uses production quick-match terminal behavior only from allowed request facts', () => { const selectedTrace = trace('knowledge-task-model'); const policy = fixedTraceHybridPolicy(); From 0497ef85ee9cbd586749b455ae51039b16a25d7a Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 22:09:15 +0000 Subject: [PATCH 16/16] fix(addie): fail closed fixed trace evidence planning --- server/src/addie/direct-tool-universe.ts | 7 +- .../eval/fixed-trace-evaluation-protocol.ts | 83 +++++--- .../eval/fixed-trace-evaluator-coordinator.ts | 104 ++++++++-- server/src/addie/eval/fixed-trace-judge.ts | 86 +++++++-- server/src/addie/eval/fixed-trace-runner.ts | 24 +++ server/src/addie/eval/fixed-trace-suite.ts | 8 + .../src/addie/eval/fixed-trace-tool-loop.ts | 19 ++ .../tests/manual/fixed-trace-provider-eval.ts | 15 +- .../unit/addie/direct-tool-universe.test.ts | 6 + .../addie/fixed-trace-diagnostic-cli.test.ts | 13 +- .../fixed-trace-evaluation-protocol.test.ts | 113 ++++++++--- .../fixed-trace-evaluator-coordinator.test.ts | 29 +++ .../unit/addie/fixed-trace-judge.test.ts | 179 ++++++++++++------ .../unit/addie/fixed-trace-runner.test.ts | 10 + 14 files changed, 534 insertions(+), 162 deletions(-) diff --git a/server/src/addie/direct-tool-universe.ts b/server/src/addie/direct-tool-universe.ts index 20ae74866c..ab2c2e4a4d 100644 --- a/server/src/addie/direct-tool-universe.ts +++ b/server/src/addie/direct-tool-universe.ts @@ -166,6 +166,7 @@ function captureFixedTraceEvaluatorToolUniverse(): CapturedDirectToolUniverse { */ export const FIXED_TRACE_DIRECT_TOOL_UNIVERSE = captureFixedTraceEvaluatorToolUniverse(); -export const FIXED_TRACE_DIRECT_TOOL_HANDLERS = createSyntheticDirectToolReceiptHandlers( - FIXED_TRACE_DIRECT_TOOL_UNIVERSE, -); +/** Construct inert handlers only when a direct replay explicitly asks for them. */ +export function fixedTraceDirectToolHandlers(): Map { + return createSyntheticDirectToolReceiptHandlers(FIXED_TRACE_DIRECT_TOOL_UNIVERSE); +} diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts index 79224508a2..63e9ae87f5 100644 --- a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -32,6 +32,7 @@ import { FIXED_TRACE_PARTITION_MANIFEST, assertFixedTracePartitionManifest, } from "./fixed-trace-partition.js"; +import { snapshotFixedTraceJson } from "./fixed-trace-safe-snapshot.js"; import { FIXED_TRACE_CORPUS } from "./fixed-trace-suite.js"; export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = @@ -344,6 +345,19 @@ export const FIXED_TRACE_ARCHITECTURE_CELL_TRUTH = Object.freeze({ mixedProviderCombinationsRequiringHumanOrFourthProvider: 134, }); +export const FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT = sha256( + FIXED_TRACE_ADMITTED_CELLS.map( + ({ id, role, provider, model, effort, pricingProfileId }) => ({ + id, + role, + provider, + model, + effort, + pricingProfileId, + }), + ), +); + /** USD is operational evidence, never a percentage-point quality hypothesis. */ export const FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE = Object.freeze({ status: "not_admitted_pending_complete_trusted_usage_and_predeclared_economic_margin", @@ -837,6 +851,11 @@ export interface FixedTraceArchitectureArmCallAccounting { } export interface FixedTraceScreeningResult { readonly cellId: string; + readonly role: "router" | "generation"; + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly configFingerprint: string; readonly safetyFailures: number; readonly identityFailures: number; readonly malformedFailures: number; @@ -848,26 +867,34 @@ export interface FixedTraceScreeningResult { /** Pure, predeclared elimination/halving rule; repetitions estimate stability only. */ export function selectFixedTraceScreeningSurvivors( results: readonly FixedTraceScreeningResult[], - requiredCellIds: readonly string[] = FIXED_TRACE_ADMITTED_CELLS.map( - (cell) => cell.id, - ), ): readonly string[] { - const required = new Set(requiredCellIds); - if ( - required.size !== requiredCellIds.length || - required.size === 0 || - [...required].some( - (cellId) => !FIXED_TRACE_ADMITTED_CELLS.some((cell) => cell.id === cellId), - ) - ) - throw new Error("screening required cell set is invalid"); + const snapshot = snapshotFixedTraceJson( + results, + "fixed-trace screening results", + ) as readonly FixedTraceScreeningResult[]; + const required = new Map( + FIXED_TRACE_ADMITTED_CELLS.map((cell) => [cell.id, cell]), + ); const seen = new Set(); - for (const result of results) { + if (snapshot.length !== required.size) + throw new Error("screening requires exactly one result for every supported executable cell"); + for (const result of snapshot) { + const cell = required.get(result.cellId); + if ( + !cell || + seen.has(result.cellId) || + result.role !== cell.role || + result.provider !== cell.provider || + result.model !== cell.model || + result.effort !== cell.effort || + result.configFingerprint !== FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT + ) + throw new Error("screening result has unknown, duplicate, or mismatched canonical cell identity"); if ( - !required.has(result.cellId) || - seen.has(result.cellId) + Object.keys(result).sort().join(",") !== + "cellId,configFingerprint,costUsd,effort,identityFailures,latencyMs,malformedFailures,model,provider,reliabilityFailures,role,safetyFailures,toolLoopFailures" ) - throw new Error("screening result has an unknown or duplicate cell"); + throw new Error("screening result has extra or missing fields"); if ( [ result.safetyFailures, @@ -882,9 +909,7 @@ export function selectFixedTraceScreeningSurvivors( throw new Error("screening result has invalid metrics"); seen.add(result.cellId); } - if (seen.size !== required.size) - throw new Error("screening requires exactly one result for every supported executable cell"); - const eligible = results.filter( + const eligible = snapshot.filter( (result) => result.safetyFailures === 0 && result.identityFailures === 0 && @@ -912,11 +937,25 @@ function sha256(value: unknown): string { export function fixedTraceEvaluationProtocolFingerprint( protocol: FixedTraceEvaluationProtocol, ): string { - assertFixedTraceEvaluationProtocol(protocol); - return sha256(protocol); + return sha256(validatedFixedTraceEvaluationProtocol(protocol)); } export function assertFixedTraceEvaluationProtocol( protocol: FixedTraceEvaluationProtocol, +): void { + void validatedFixedTraceEvaluationProtocol(protocol); +} +function validatedFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, +): FixedTraceEvaluationProtocol { + const snapshot = snapshotFixedTraceJson( + protocol, + "fixed-trace evaluation protocol", + ) as FixedTraceEvaluationProtocol; + validateFixedTraceEvaluationProtocol(snapshot); + return snapshot; +} +function validateFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, ): void { assertFixedTracePartitionManifest(); if ( @@ -1067,7 +1106,7 @@ export function assertFixedTraceEvaluationProtocol( export function estimateFixedTraceEvaluationProtocol( protocol: FixedTraceEvaluationProtocol = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, ): FixedTraceProtocolEstimate { - assertFixedTraceEvaluationProtocol(protocol); + protocol = validatedFixedTraceEvaluationProtocol(protocol); const stages: FixedTraceStageCeiling[] = []; for (const phase of protocol.phases) for (const arm of phase.arms) { diff --git a/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts index 73a2c60c18..6a37fe8de1 100644 --- a/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts +++ b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts @@ -1,8 +1,10 @@ import { createHmac, timingSafeEqual } from "node:crypto"; +import { types } from "node:util"; import type { ModelProviderId, ModelReasoningEffort, } from "../model-providers/model-provider.js"; +import { snapshotFixedTraceJson } from "./fixed-trace-safe-snapshot.js"; /** * Diagnostic integrity only. An importer supplies this module's key, so this @@ -122,6 +124,10 @@ export interface FixedTraceEvidenceLedger { const DIAGNOSTIC_ADMISSION = "not_admitted_diagnostic_hmac_without_privileged_durable_authority" as const; +const hasExactKeys = (value: unknown, keys: readonly string[]) => + typeof value === "object" && + value !== null && + Object.keys(value).sort().join(",") === [...keys].sort().join(","); function canonical(value: unknown): string { if (value === null || typeof value === "boolean" || typeof value === "string") @@ -142,13 +148,71 @@ function canonical(value: unknown): string { throw new Error("non-JSON evaluator ledger value"); } function deepSnapshot(value: T): T { - const copy = structuredClone(value); - const freeze = (nested: unknown): unknown => { - if (nested === null || typeof nested !== "object" || Object.isFrozen(nested)) return nested; - for (const child of Object.values(nested as Record)) freeze(child); - return Object.freeze(nested); - }; - return freeze(copy) as T; + return snapshotFixedTraceJson(value, "fixed-trace evaluator coordinator") as T; +} +function snapshotCoordinatorConfig(value: unknown): { + readonly hmacKey: Uint8Array; + readonly keyId: string; +} { + if ( + typeof value !== "object" || + value === null || + types.isProxy(value) || + Object.getPrototypeOf(value) !== Object.prototype || + Object.getOwnPropertySymbols(value).length !== 0 + ) + throw new Error("evaluator coordinator configuration must be a plain non-proxy object"); + const descriptors = Object.getOwnPropertyDescriptors(value); + if (Object.keys(descriptors).sort().join(",") !== "hmacKey,keyId") + throw new Error("evaluator coordinator configuration has extra or missing fields"); + const key = descriptors.hmacKey; + const keyId = descriptors.keyId; + if (!key || !("value" in key) || !keyId || !("value" in keyId)) + throw new Error("evaluator coordinator configuration must use data properties"); + if ( + types.isProxy(key.value) || + !(key.value instanceof Uint8Array) || + key.value.byteLength < 32 || + typeof keyId.value !== "string" || + !keyId.value.trim() + ) + throw new Error("evaluator-owned HMAC custody configuration is required"); + return Object.freeze({ hmacKey: new Uint8Array(key.value), keyId: keyId.value }); +} +const isProvider = (value: unknown): value is ModelProviderId => + value === "anthropic" || value === "openai" || value === "google"; +const isEffort = (value: unknown): value is ModelReasoningEffort => + value === "provider_default" || value === "none" || value === "low" || value === "medium" || value === "high"; +const isNonemptyString = (value: unknown): value is string => + typeof value === "string" && value.trim().length > 0; +function assertExpectedInvocation(entry: FixedTraceExpectedInvocation, runId: string): void { + if (!hasExactKeys(entry, [ + "runId", "phaseId", "caseId", "armId", "stage", "invocation", "attempt", "requested", "controls", + ])) throw new Error("expected sequence entry has extra or missing fields"); + if ( + entry.runId !== runId || + !isNonemptyString(entry.phaseId) || + !isNonemptyString(entry.caseId) || + !isNonemptyString(entry.armId) || + !["router", "generation", "judge", "simulator"].includes(entry.stage) || + !Number.isSafeInteger(entry.invocation) || entry.invocation < 1 || + !Number.isSafeInteger(entry.attempt) || entry.attempt < 1 + ) throw new Error("expected sequence entry has invalid cross-field identity"); + if (!hasExactKeys(entry.requested, ["provider", "model", "effort", "identityPolicy"]) || + !isProvider(entry.requested.provider) || !isNonemptyString(entry.requested.model) || + !isEffort(entry.requested.effort) || !isNonemptyString(entry.requested.identityPolicy)) + throw new Error("expected sequence entry has invalid requested identity"); + const controls = entry.controls; + if (!hasExactKeys(controls, [ + "promptSha256", "systemSha256", "messagesSha256", "toolSchemaSha256", "providerRequestSha256", + "presentedToolNames", "presentedToolOrderSha256", "simulatorReceiptProvenanceSha256", + "simulatorControlsSha256", "architectureSha256", "admissionSha256", "configSha256", "pricingSha256", + "limitsSha256", "retryCacheSamplingSha256", "failureDenominatorId", + ]) || + !Object.entries(controls).every(([key, value]) => key === "presentedToolNames" + ? Array.isArray(value) && value.every(isNonemptyString) + : isNonemptyString(value))) + throw new Error("expected sequence entry has invalid controls"); } const invocationKey = ( entry: Pick< @@ -212,19 +276,21 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { readonly hmacKey: Uint8Array; readonly keyId: string; }) { - if ( - !(evaluatorConfig.hmacKey instanceof Uint8Array) || - evaluatorConfig.hmacKey.byteLength < 32 || - !evaluatorConfig.keyId.trim() - ) - throw new Error("evaluator-owned HMAC custody configuration is required"); + const detachedConfig = snapshotCoordinatorConfig(evaluatorConfig); const sign = (projection: string) => - createHmac("sha256", evaluatorConfig.hmacKey) + createHmac("sha256", detachedConfig.hmacKey) .update( - `${FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION}\u0000${evaluatorConfig.keyId}\u0000${projection}`, + `${FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION}\u0000${detachedConfig.keyId}\u0000${projection}`, ) .digest("hex"); const verify = (contract: FixedTraceExpectedSequenceContract) => { + if (!hasExactKeys(contract, [ + "version", "keyId", "runId", "protocolFingerprint", "manifestFingerprint", "entries", "signature", + ])) + throw new FixedTraceLedgerValidationError( + "authentication", + "expected sequence contract has extra or missing fields", + ); const projection = contractProjection({ version: contract.version, keyId: contract.keyId, @@ -237,7 +303,7 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { const supplied = Buffer.from(contract.signature, "hex"); if ( contract.version !== FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION || - contract.keyId !== evaluatorConfig.keyId || + contract.keyId !== detachedConfig.keyId || expected.length !== supplied.length || !timingSafeEqual(expected, supplied) ) @@ -254,6 +320,9 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { "version" | "keyId" | "signature" >, ): FixedTraceExpectedSequenceContract { + input = deepSnapshot(input); + if (!hasExactKeys(input, ["runId", "protocolFingerprint", "manifestFingerprint", "entries"])) + throw new Error("expected sequence has extra or missing fields"); if ( !input.runId.trim() || !input.protocolFingerprint.trim() || @@ -265,6 +334,7 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { ); const keys = new Set(); for (const entry of input.entries) { + assertExpectedInvocation(entry, input.runId); const key = invocationKey(entry); if (entry.runId !== input.runId || keys.has(key)) throw new Error( @@ -274,7 +344,7 @@ export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { } const unsigned = deepSnapshot({ version: FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION, - keyId: evaluatorConfig.keyId, + keyId: detachedConfig.keyId, ...input, entries: deepSnapshot(input.entries), } as const); diff --git a/server/src/addie/eval/fixed-trace-judge.ts b/server/src/addie/eval/fixed-trace-judge.ts index cf2fb88fab..73fa3cabff 100644 --- a/server/src/addie/eval/fixed-trace-judge.ts +++ b/server/src/addie/eval/fixed-trace-judge.ts @@ -22,9 +22,20 @@ import type { export const FIXED_TRACE_JUDGE_PROMPT_VERSION = 'addie-fixed-trace-blinded-judge-v2'; export const FIXED_TRACE_MIN_INDEPENDENT_JUDGES = 2; +/** + * There is no privileged calibration custody boundary in this integration + * draft. A caller-provided hash or boolean cannot satisfy this admission. + */ +export const FIXED_TRACE_JUDGE_CALIBRATION_ADMISSION = + 'not_admitted_missing_privileged_custodied_calibration' as const; +function hasPrivilegedCustodiedCalibration(): boolean { + return false; +} const MAX_JUDGE_INPUT_BYTES = 24 * 1024; const MAX_JUDGE_OUTPUT_BYTES = 8 * 1024; +const isModelProviderId = (value: unknown): value is ModelProviderId => + value === 'anthropic' || value === 'openai' || value === 'google'; const FIXED_TRACE_JUDGE_VERDICT_SCHEMA: Readonly = Object.freeze({ type: 'object', properties: { @@ -65,6 +76,7 @@ export type FixedTraceJudgeStatus = export type FixedTraceJudgeFailureReason = | 'candidate_not_judgeable' | 'judge_not_independent' + | 'judge_calibration_not_admitted' | 'judge_input_out_of_bounds' | 'judge_output_truncated' | 'judge_output_invalid' @@ -331,16 +343,47 @@ function validateConfig(config: FixedTraceJudgeConfig): void { ) throw new Error('Judge pricing is invalid'); } -function candidateProviders(observation: FixedTraceObservation): ReadonlySet { - // A router selects the candidate-visible surface, so both stages contribute - // to the pipeline under semantic review—even when generation metadata exists - // or a fallback changed the returned provider. - return new Set([ - observation.metadata.router.requestedProvider, - observation.metadata.router.returnedProvider, - observation.metadata.generation.requestedProvider, - observation.metadata.generation.returnedProvider, - ].filter((provider): provider is ModelProviderId => provider !== null)); +function candidateProviders( + observation: FixedTraceObservation, +): ReadonlySet | null { + const stages = [observation.metadata.router, observation.metadata.generation]; + const providers = new Set(); + for (const stage of stages) { + if (!stage.providerExposures) return null; + if (stage.source === 'provider' && stage.providerExposures.length === 0) return null; + if (stage.providerExposures.length !== stage.dispatchedCalls) return null; + const attempts = new Set(); + const preparedIdentities = new Set(); + const returnedIdentities = new Set(); + for (const exposure of stage.providerExposures) { + if ( + !Number.isSafeInteger(exposure.attempt) || + exposure.attempt < 1 || + !exposure.preparedModel || + exposure.attempt > stage.dispatchedCalls || + !isModelProviderId(exposure.preparedProvider) || + (exposure.returnedProvider === null) !== (exposure.returnedModel === null) || + (exposure.returnedProvider !== null && !isModelProviderId(exposure.returnedProvider)) || + (exposure.returnedModel !== null && !exposure.returnedModel) + ) return null; + const preparedIdentity = `${exposure.preparedProvider}\u0000${exposure.preparedModel}`; + if (attempts.has(exposure.attempt)) return null; + attempts.add(exposure.attempt); + preparedIdentities.add(preparedIdentity); + providers.add(exposure.preparedProvider); + if (exposure.returnedProvider) { + returnedIdentities.add(`${exposure.returnedProvider}\u0000${exposure.returnedModel}`); + providers.add(exposure.returnedProvider); + } + } + if ( + (stage.requestedProvider === null) !== (stage.requestedModel === null) || + (stage.returnedProvider === null) !== (stage.returnedModel === null) || + (stage.requestedProvider !== null && !preparedIdentities.has(`${stage.requestedProvider}\u0000${stage.requestedModel}`)) || + (stage.returnedProvider !== null && !returnedIdentities.has(`${stage.returnedProvider}\u0000${stage.returnedModel}`)) + ) return null; + } + return providers.size ? providers : null; } export async function judgeFixedTraceObservation( @@ -374,7 +417,7 @@ export async function judgeFixedTraceObservation( if ( !trace.answerRubric?.length || observation.terminalStatus !== 'complete' - || candidateProviderIds.size === 0 + || candidateProviderIds === null ) { return { traceId: trace.id, @@ -393,6 +436,18 @@ export async function judgeFixedTraceObservation( metadata: metadata(config, request, [], false, startedAt, null), }; } + // Do not let a planning-side calibration record authorize a provider call. + // A separately injected privileged, custodied calibration verifier is the + // prerequisite; this module intentionally has no caller-mintable seam. + if (!hasPrivilegedCustodiedCalibration()) { + return { + traceId: trace.id, + status: 'skipped', + failureReason: 'judge_calibration_not_admitted', + verdict: null, + metadata: metadata(config, request, [], false, startedAt, null), + }; + } const invocations: PreparedModelInvocation[] = []; let dispatched = false; @@ -458,6 +513,8 @@ export async function runIndependentFixedTraceJudges( observations: ReadonlyArray, judgeConfigs: ReadonlyArray, ): Promise { + if (!hasPrivilegedCustodiedCalibration()) + throw new Error('independent judge dispatch is not admitted without privileged custodied calibration'); const configsByProvider = new Map(); for (const config of judgeConfigs) { if (configsByProvider.has(config.provider.id)) throw new Error('Independent judges must use unique providers'); @@ -469,6 +526,8 @@ export async function runIndependentFixedTraceJudges( const observation = observationsById.get(trace.id); if (!observation) continue; const candidateProviderIds = candidateProviders(observation); + if (!candidateProviderIds) + throw new Error(`Trace ${trace.id} has incomplete candidate provider exposure`); const independentConfigs = judgeConfigs.filter((config) => !candidateProviderIds.has(config.provider.id)); if (independentConfigs.length < FIXED_TRACE_MIN_INDEPENDENT_JUDGES) { throw new Error(`Trace ${trace.id} requires at least two independent judge providers`); @@ -504,9 +563,11 @@ export function summarizeFixedTraceJudges( for (const trace of applicable) { const group = byTrace.get(trace.id) ?? []; const providers = new Set(group.map((judgment) => judgment.metadata.requestedProvider)); - const candidates = candidateProviderIds.get(trace.id) ?? new Set(); + const candidates = candidateProviderIds.get(trace.id); const complete = group.length >= FIXED_TRACE_MIN_INDEPENDENT_JUDGES && providers.size === group.length + && candidates !== null + && candidates !== undefined && candidates.size > 0 && [...providers].every((provider) => !candidates.has(provider)) && group.every((judgment) => judgment.status === 'judged' && judgment.verdict !== null); @@ -527,6 +588,7 @@ export function summarizeFixedTraceJudges( const ratio = (count: number, denominator: number) => denominator === 0 ? 0 : count / denominator; const judgedJudgments = judgments.filter((judgment) => judgment.status === 'judged').length; const comparisonEligible = applicable.length > 0 + && hasPrivilegedCustodiedCalibration() && completeCases.every(Boolean) && judgments.length === expectedJudgments && totalEstimatedCostUsd !== null; diff --git a/server/src/addie/eval/fixed-trace-runner.ts b/server/src/addie/eval/fixed-trace-runner.ts index 632f903b7d..f42b6c56a8 100644 --- a/server/src/addie/eval/fixed-trace-runner.ts +++ b/server/src/addie/eval/fixed-trace-runner.ts @@ -332,6 +332,25 @@ interface StageInvocationState { latencyMs: number; } +function providerExposures( + state: StageInvocationState, + response?: ModelResponse, + recordedExposures?: NonNullable, +): FixedTraceModelStageMetadata["providerExposures"] { + if (recordedExposures) return deepFreeze(recordedExposures.map((exposure) => ({ ...exposure }))); + return deepFreeze( + state.invocations.map((prepared, index) => ({ + attempt: index + 1, + preparedProvider: prepared.provider, + preparedModel: prepared.model, + returnedProvider: + response && index === state.invocations.length - 1 ? response.provider : null, + returnedModel: + response && index === state.invocations.length - 1 ? response.model : null, + })), + ); +} + function canonicalJson(value: unknown): string { if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); if (typeof value === 'number') { @@ -459,6 +478,7 @@ function providerStageMetadata( response: ModelResponse, usage: ModelUsage, state: StageInvocationState, + recordedExposures?: NonNullable, ): FixedTraceModelStageMetadata { // Provider responses are outside evaluator ownership. Retaining their usage // object would let a later provider turn mutate already-recorded cost and @@ -473,6 +493,7 @@ function providerStageMetadata( requestedModel: config.model, returnedProvider: response.provider, returnedModel: response.model, + providerExposures: providerExposures(state, response, recordedExposures), modelResolution: modelResolution(config, response), promptSha256: promptSha256(request), providerRequestSha256: providerRequestSha256(state.invocations), @@ -510,6 +531,7 @@ function localStageMetadata( requestedModel: config.model, returnedProvider: null, returnedModel: null, + providerExposures: providerExposures(state), modelResolution: 'local', promptSha256: promptSha256(request), providerRequestSha256: providerRequestSha256(state.invocations), @@ -540,6 +562,7 @@ function notRunStageMetadata(trace: FixedTraceCase): FixedTraceModelStageMetadat requestedModel: null, returnedProvider: null, returnedModel: null, + providerExposures: Object.freeze([]), modelResolution: null, promptSha256: null, providerRequestSha256: null, @@ -1122,6 +1145,7 @@ export async function runFixedTraceCase( result.response, result.usage, state, + result.providerExposures, ); const terminalStatus = terminalStatusForFinishReason(result.response.finishReason, result.text); return { diff --git a/server/src/addie/eval/fixed-trace-suite.ts b/server/src/addie/eval/fixed-trace-suite.ts index 7cee007a7f..cce17d6b77 100644 --- a/server/src/addie/eval/fixed-trace-suite.ts +++ b/server/src/addie/eval/fixed-trace-suite.ts @@ -305,6 +305,14 @@ export interface FixedTraceModelStageMetadata { requestedModel: string | null; returnedProvider: ModelProviderId | null; returnedModel: string | null; + /** Identity-only ledger of every prepared attempt; never contains payloads. */ + providerExposures?: readonly { + attempt: number; + preparedProvider: ModelProviderId; + preparedModel: string; + returnedProvider: ModelProviderId | null; + returnedModel: string | null; + }[]; modelResolution: 'exact' | 'provider_canonicalized' | 'local' | null; promptSha256: string | null; providerRequestSha256: string | null; diff --git a/server/src/addie/eval/fixed-trace-tool-loop.ts b/server/src/addie/eval/fixed-trace-tool-loop.ts index 20c11fab88..178d33920b 100644 --- a/server/src/addie/eval/fixed-trace-tool-loop.ts +++ b/server/src/addie/eval/fixed-trace-tool-loop.ts @@ -64,6 +64,14 @@ export interface FixedTraceToolLoopResult { usage: ModelUsage; tools: ReadonlyArray; invocations: ReadonlyArray; + /** Identity-only record for each dispatched model turn; never prompt data. */ + providerExposures: ReadonlyArray<{ + attempt: number; + preparedProvider: PreparedModelInvocation['provider']; + preparedModel: string; + returnedProvider: ModelResponse['provider']; + returnedModel: string; + }>; } export interface FixedTraceToolLoopOptions { @@ -291,6 +299,7 @@ export async function executeFixedTraceToolLoop( const executions: FixedTraceToolExecution[] = []; const completedExecutions: ToolExecution[] = []; const invocations: PreparedModelInvocation[] = []; + const providerExposures: FixedTraceToolLoopResult['providerExposures'][number][] = []; const seenCallIds = new Set(); const seenToolNames = new Set(); const modelLoop = new ModelTurnLoopState(iterationLimit); @@ -328,6 +337,15 @@ export async function executeFixedTraceToolLoop( await options.beforeDispatch?.(prepared); }, }); + const prepared = invocations.at(-1); + if (!prepared) throw new Error('fixed-trace model response was not preceded by a prepared invocation'); + providerExposures.push(Object.freeze({ + attempt: invocations.length, + preparedProvider: prepared.provider, + preparedModel: prepared.model, + returnedProvider: response.provider, + returnedModel: response.model, + })); const turn = activeTurn.acceptResponse(response); if (turn.providerToolCalls.length > 0 || turn.providerToolResults.length > 0) { @@ -350,6 +368,7 @@ export async function executeFixedTraceToolLoop( usage: modelLoop.usage, tools: Object.freeze([...executions]), invocations: Object.freeze([...invocations]), + providerExposures: Object.freeze([...providerExposures]), }; } diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index f77e260424..7f7d046cd5 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -1,10 +1,5 @@ /** Planning-only manual entrypoint: it has no dispatch or output path. */ import { parseFixedTraceDiagnosticCliArguments } from "../../src/addie/eval/fixed-trace-diagnostic-cli.js"; -import { - FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, - assertFixedTraceEvaluationProtocol, - estimateFixedTraceEvaluationProtocol, -} from "../../src/addie/eval/fixed-trace-evaluation-protocol.js"; const arguments_ = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); if (!arguments_.validateOnly) @@ -15,6 +10,16 @@ if (arguments_.output !== undefined) throw new Error( "--output is unavailable in validate-only mode; no artifact may be written", ); +// This entrypoint is deliberately data-only. Some transitive corpus modules +// initialize diagnostic loggers while their immutable declarations load; keep +// those process-local diagnostics isolated from the one-machine-readable-line +// validate-only contract. +process.env.LOG_LEVEL = "silent"; +const { + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + assertFixedTraceEvaluationProtocol, + estimateFixedTraceEvaluationProtocol, +} = await import("../../src/addie/eval/fixed-trace-evaluation-protocol.js"); assertFixedTraceEvaluationProtocol(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); const estimate = estimateFixedTraceEvaluationProtocol( FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, diff --git a/server/tests/unit/addie/direct-tool-universe.test.ts b/server/tests/unit/addie/direct-tool-universe.test.ts index 3d708ea5c5..e1f0358c4f 100644 --- a/server/tests/unit/addie/direct-tool-universe.test.ts +++ b/server/tests/unit/addie/direct-tool-universe.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { createSyntheticDirectToolReceiptHandlers, FIXED_TRACE_DIRECT_TOOL_UNIVERSE, + fixedTraceDirectToolHandlers, type CapturedDirectToolUniverse, } from '../../../src/addie/direct-tool-universe.js'; import { getSafeReadOnlyFallbackTools } from '../../../src/addie/tool-sets.js'; @@ -53,4 +54,9 @@ describe('direct tool-universe evaluator descriptors', () => { }); expect(mockHandler).not.toHaveBeenCalled(); }); + + it('constructs synthetic handlers only through the explicit replay factory', () => { + const handlers = fixedTraceDirectToolHandlers(); + expect([...handlers.keys()]).toEqual(FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolNames); + }); }); diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts index cafcd8ed56..5459efc9ea 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts @@ -51,16 +51,9 @@ describe("fixed-trace diagnostic CLI parser", () => { env: { PATH: process.env.PATH ?? "" }, }, ); - const validated = result - .split("\n") - .map((line) => { - try { - return JSON.parse(line) as Record; - } catch { - return null; - } - }) - .find((line) => line?.diagnosticOnly === true); + const lines = result.trim().split("\n"); + expect(lines).toHaveLength(1); + const validated = JSON.parse(lines[0]!) as Record; expect(validated).toMatchObject({ diagnosticOnly: true, dispatchable: false, diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts index 51a93a8004..0f42050bef 100644 --- a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -10,10 +10,12 @@ import { FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE, FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, FIXED_TRACE_PROTOCOL_PRICING, + FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT, FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, assertPromotionGradeDualJudgeFeasibility, assertFixedTraceEvaluationProtocol, estimateFixedTraceEvaluationProtocol, + fixedTraceEvaluationProtocolFingerprint, providerExcludingCalibratedJudges, selectFixedTraceScreeningSurvivors, semanticJudgeCandidateProviders, @@ -24,6 +26,22 @@ import { } from "../../../src/addie/eval/fixed-trace-partition.js"; import { resolveModelCostPricing } from "../../../src/addie/model-cost-pricing.js"; +const screeningResult = (cell = FIXED_TRACE_ADMITTED_CELLS[0]!, index = 0) => ({ + cellId: cell.id, + role: cell.role, + provider: cell.provider, + model: cell.model, + effort: cell.effort, + configFingerprint: FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT, + safetyFailures: 0, + identityFailures: 0, + malformedFailures: 0, + toolLoopFailures: 0, + reliabilityFailures: index, + latencyMs: 100 + index, + costUsd: index, +}); + describe("fixed-trace staged protocol", () => { it("derives the complete 46 development / 36 tuning partitions from corpus authority", () => { assertFixedTracePartitionManifest(); @@ -270,46 +288,48 @@ describe("fixed-trace staged protocol", () => { ); }); it("applies hard elimination and successive halving deterministically", () => { - const results = FIXED_TRACE_ADMITTED_CELLS.slice(0, 4).map( - (cell, index) => ({ - cellId: cell.id, - safetyFailures: index === 3 ? 1 : 0, - identityFailures: 0, - malformedFailures: 0, - toolLoopFailures: 0, - reliabilityFailures: index, - latencyMs: 10 - index, - costUsd: index, - }), + const results = FIXED_TRACE_ADMITTED_CELLS.map(screeningResult); + results[20]!.safetyFailures = 1; + expect(selectFixedTraceScreeningSurvivors(results)).toEqual( + results.slice(0, 10).map((result) => result.cellId), + ); + expect(selectFixedTraceScreeningSurvivors([...results].reverse())).toEqual( + results.slice(0, 10).map((result) => result.cellId), ); - const required = results.map((result) => result.cellId); - expect(selectFixedTraceScreeningSurvivors(results, required)).toEqual([ - results[0]!.cellId, - results[1]!.cellId, - ]); - expect(selectFixedTraceScreeningSurvivors([...results].reverse(), required)).toEqual([ - results[0]!.cellId, - results[1]!.cellId, - ]); }); it("rejects partial and hostile screening result sets", () => { - const result = { - cellId: FIXED_TRACE_ADMITTED_CELLS[0]!.id, - safetyFailures: 0, identityFailures: 0, malformedFailures: 0, - toolLoopFailures: 0, reliabilityFailures: 0, latencyMs: 1, costUsd: 1, - }; + const result = screeningResult(); expect(() => selectFixedTraceScreeningSurvivors([result])).toThrow( "exactly one result for every supported executable cell", ); + const complete = FIXED_TRACE_ADMITTED_CELLS.map(screeningResult); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, latencyMs: Number.NaN } : entry), + )).toThrow("non-finite number"); + expect(() => selectFixedTraceScreeningSurvivors( + [...complete.slice(0, -1), complete[0]!], + )).toThrow("unknown, duplicate, or mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, provider: "google" } : entry), + )).toThrow("mismatched canonical cell identity"); expect(() => selectFixedTraceScreeningSurvivors( - [{ ...result, latencyMs: Number.NaN }], [result.cellId], - )).toThrow("invalid metrics"); + complete.map((entry, index) => index === 0 ? { ...entry, role: "generation" } : entry), + )).toThrow("mismatched canonical cell identity"); expect(() => selectFixedTraceScreeningSurvivors( - [result, result], [result.cellId], - )).toThrow("unknown or duplicate"); + complete.map((entry, index) => index === 0 ? { ...entry, model: "forged-model" } : entry), + )).toThrow("mismatched canonical cell identity"); expect(() => selectFixedTraceScreeningSurvivors( - [result], ["unknown"], - )).toThrow("required cell set is invalid"); + complete.map((entry, index) => index === 0 ? { ...entry, effort: "high" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, cellId: "alias" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, configFingerprint: "forged" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, forged: true } : entry), + )).toThrow("extra or missing fields"); }); it.each([ (protocol: any) => { protocol.finalProtocol.familywiseAlpha = 0.5; }, @@ -324,6 +344,37 @@ describe("fixed-trace staged protocol", () => { mutate(protocol); expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow(); }); + it("rejects getters and proxies before protocol validation, fingerprinting, or budgeting", () => { + const getterProtocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + Object.defineProperty(getterProtocol.adaptiveRule, "repeats", { + enumerable: true, + get: () => "stability_only_not_new_cases", + }); + for (const action of [ + () => assertFixedTraceEvaluationProtocol(getterProtocol), + () => fixedTraceEvaluationProtocolFingerprint(getterProtocol), + () => estimateFixedTraceEvaluationProtocol(getterProtocol), + ]) expect(action).toThrow("own enumerable data property"); + const proxy = new Proxy(structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL), {}); + expect(() => estimateFixedTraceEvaluationProtocol(proxy)).toThrow("must not contain a Proxy"); + const togglingProtocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + let reads = 0; + Object.defineProperty(togglingProtocol.adaptiveRule, "repeats", { + enumerable: true, + get: () => (++reads === 1 ? "stability_only_not_new_cases" : "999"), + }); + expect(() => fixedTraceEvaluationProtocolFingerprint(togglingProtocol)) + .toThrow("own enumerable data property"); + expect(reads).toBe(0); + }); + it("uses a detached protocol snapshot rather than a later nested mutation", () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + const estimate = estimateFixedTraceEvaluationProtocol(protocol); + protocol.phases[5].repetitions = 999; + expect(estimate.stages.find((stage) => stage.phaseId === "stage_4_tuning")?.calls) + .not.toBe(36 * 999); + expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow("pinned declaration"); + }); it("uses canonical Luna subset-cache pricing and leaves Terra/Sol inert", () => { const luna = FIXED_TRACE_PROTOCOL_PRICING.find( (profile) => profile.provider === "openai", diff --git a/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts index f95d4fa8d7..c2304dca83 100644 --- a/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts +++ b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts @@ -130,6 +130,35 @@ describe("fixed-trace evaluator-owned evidence coordinator", () => { expect(arbitraryImporter.validate(issued, [actual(issued.entries[0]!)]).admission) .toBe("not_admitted_diagnostic_hmac_without_privileged_durable_authority"); }); + it("rejects getter/proxy inputs and detaches mutable key material", () => { + const key = new Uint8Array(32).fill(3); + const config = { hmacKey: key, keyId: "detached-key" }; + const detached = createFixedTraceEvaluatorCoordinator(config); + key.fill(4); + config.keyId = "rewritten-key"; + const issued = detached.issueExpectedSequence({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", + entries: [expected("case-a", 1)], + }); + expect(issued.keyId).toBe("detached-key"); + expect(detached.validate(issued, [actual(issued.entries[0]!)]).complete).toBe(true); + const getterInput = { + protocolFingerprint: "protocol", manifestFingerprint: "manifest", entries: [expected("case-a", 1)], + } as Record; + let reads = 0; + Object.defineProperty(getterInput, "runId", { + enumerable: true, + get: () => (++reads === 1 ? "run-1" : "run-2"), + }); + expect(() => detached.issueExpectedSequence(getterInput as any)).toThrow("own enumerable data property"); + expect(reads).toBe(0); + expect(() => createFixedTraceEvaluatorCoordinator(new Proxy(config, {}))).toThrow("non-proxy"); + expect(() => detached.issueExpectedSequence(new Proxy({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", entries: [expected("case-a", 1)], + }, {}))).toThrow("must not contain a Proxy"); + const actualEntries = [actual(issued.entries[0]!)]; + expect(() => detached.validate(issued, new Proxy(actualEntries, {}))).toThrow("must not contain a Proxy"); + }); it.each([ [ "omission", diff --git a/server/tests/unit/addie/fixed-trace-judge.test.ts b/server/tests/unit/addie/fixed-trace-judge.test.ts index 40fe7c3821..52656b14c6 100644 --- a/server/tests/unit/addie/fixed-trace-judge.test.ts +++ b/server/tests/unit/addie/fixed-trace-judge.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { - FIXED_TRACE_MIN_INDEPENDENT_JUDGES, + FIXED_TRACE_JUDGE_CALIBRATION_ADMISSION, buildFixedTraceJudgeRequest, judgeFixedTraceObservation, runIndependentFixedTraceJudges, @@ -124,10 +124,18 @@ function stage(provider: ModelProviderId): FixedTraceModelStageMetadata { return { source: "provider", dispatched: true, + dispatchedCalls: 1, requestedProvider: provider, requestedModel: `${provider}-candidate-secret-model`, returnedProvider: provider, returnedModel: `${provider}-candidate-secret-model`, + providerExposures: [{ + attempt: 1, + preparedProvider: provider, + preparedModel: `${provider}-candidate-secret-model`, + returnedProvider: provider, + returnedModel: `${provider}-candidate-secret-model`, + }], modelResolution: "exact", promptSha256: "a".repeat(64), providerRequestSha256: "b".repeat(64), @@ -234,7 +242,7 @@ describe("fixed-trace independent judge", () => { }); }); - it("accepts a strict, internally consistent verdict with complete provenance", async () => { + it("does not dispatch even a strict verdict without custodied calibration", async () => { const provider = new ScriptedJudgeProvider( "openai", '{"pass":true,"score":4,"reason":"correct","finding":"The answer matches the executed tool evidence."}', @@ -245,32 +253,27 @@ describe("fixed-trace independent judge", () => { config(provider), ); expect(result).toMatchObject({ - status: "judged", - failureReason: null, - verdict: { - pass: true, - score: 4, - reason: "correct", - finding: "The answer matches the executed tool evidence.", - }, + status: "skipped", + failureReason: "judge_calibration_not_admitted", + verdict: null, metadata: { candidateIdentityMetadataExposed: false, requestedProvider: "openai", - returnedProvider: "openai", - usageKnown: true, + returnedProvider: null, + usageKnown: false, maxIterations: 1, transportRetries: 0, samplingMode: "provider_no_sampling_control", temperature: null, }, }); - expect(result.metadata.promptSha256).toMatch(/^[a-f0-9]{64}$/); - expect(result.metadata.providerRequestSha256).toMatch(/^[a-f0-9]{64}$/); - expect(result.metadata.responseSha256).toMatch(/^[a-f0-9]{64}$/); - expect(result.metadata.estimatedCostUsd).toBeCloseTo(0.000044); + expect(provider.dispatches).toBe(0); + expect(FIXED_TRACE_JUDGE_CALIBRATION_ADMISSION).toBe( + "not_admitted_missing_privileged_custodied_calibration", + ); }); - it("joins a valid verdict split across provider text blocks", async () => { + it("does not process provider text before calibration admission", async () => { const provider = new ScriptedJudgeProvider("openai", [ '{"pass":true,', '"score":3,"reason":"correct","finding":"The answer is supported."}', @@ -282,12 +285,12 @@ describe("fixed-trace independent judge", () => { config(provider), ), ).resolves.toMatchObject({ - status: "judged", - verdict: { pass: true, score: 3, reason: "correct" }, + status: "skipped", + failureReason: "judge_calibration_not_admitted", }); }); - it("accepts a verdict accompanied only by authenticated provider thinking state", async () => { + it("does not process provider state before calibration admission", async () => { const provider = new ScriptedJudgeProvider( "anthropic", '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', @@ -301,12 +304,12 @@ describe("fixed-trace independent judge", () => { config(provider), ), ).resolves.toMatchObject({ - status: "judged", - verdict: { pass: true, score: 4, reason: "correct" }, + status: "skipped", + failureReason: "judge_calibration_not_admitted", }); }); - it("rejects inconsistent or truncated judge output", async () => { + it("does not dispatch malformed candidate verdicts before calibration admission", async () => { const inconsistent = new ScriptedJudgeProvider( "openai", '{"pass":true,"score":2,"reason":"correct"}', @@ -323,8 +326,8 @@ describe("fixed-trace independent judge", () => { config(inconsistent), ), ).resolves.toMatchObject({ - status: "invalid", - failureReason: "judge_output_invalid", + status: "skipped", + failureReason: "judge_calibration_not_admitted", }); await expect( judgeFixedTraceObservation( @@ -333,12 +336,12 @@ describe("fixed-trace independent judge", () => { config(truncated), ), ).resolves.toMatchObject({ - status: "invalid", - failureReason: "judge_output_truncated", + status: "skipped", + failureReason: "judge_calibration_not_admitted", }); }); - it("requires a bounded audit finding in every verdict", async () => { + it("does not dispatch malformed audit findings before calibration admission", async () => { const missing = new ScriptedJudgeProvider( "openai", '{"pass":true,"score":4,"reason":"correct"}', @@ -364,8 +367,8 @@ describe("fixed-trace independent judge", () => { config(provider), ), ).resolves.toMatchObject({ - status: "invalid", - failureReason: "judge_output_invalid", + status: "skipped", + failureReason: "judge_calibration_not_admitted", }); } }); @@ -393,6 +396,13 @@ describe("fixed-trace independent judge", () => { candidate.metadata.generation.returnedModel = "google-fallback-secret-model"; candidate.metadata.generation.modelResolution = "provider_canonicalized"; + candidate.metadata.generation.providerExposures = [{ + attempt: 1, + preparedProvider: "anthropic", + preparedModel: "anthropic-candidate-secret-model", + returnedProvider: "google", + returnedModel: "google-fallback-secret-model", + }]; const provider = new ScriptedJudgeProvider( "google", '{"pass":true,"score":4,"reason":"correct"}', @@ -412,10 +422,26 @@ describe("fixed-trace independent judge", () => { it("unions requested and returned router and generator providers for pipeline exclusion", async () => { const candidate = observation(trace.id, "anthropic"); candidate.metadata.router.returnedProvider = "openai"; + candidate.metadata.router.requestedModel = "anthropic-router"; candidate.metadata.router.returnedModel = "openai-router-fallback"; candidate.metadata.generation.requestedProvider = "google"; + candidate.metadata.generation.requestedModel = "google-generator"; candidate.metadata.generation.returnedProvider = "google"; candidate.metadata.generation.returnedModel = "google-generator-fallback"; + candidate.metadata.router.providerExposures = [{ + attempt: 1, + preparedProvider: "anthropic", + preparedModel: "anthropic-router", + returnedProvider: "openai", + returnedModel: "openai-router-fallback", + }]; + candidate.metadata.generation.providerExposures = [{ + attempt: 1, + preparedProvider: "google", + preparedModel: "google-generator", + returnedProvider: "google", + returnedModel: "google-generator-fallback", + }]; const onlyRemainingProvider = new ScriptedJudgeProvider( "openai", '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', @@ -428,12 +454,47 @@ describe("fixed-trace independent judge", () => { .resolves.toMatchObject({ status: "skipped", failureReason: "judge_not_independent" }); await expect(runIndependentFixedTraceJudges( [trace], [candidate], [config(onlyRemainingProvider)], - )).rejects.toThrow("requires at least two independent judge providers"); + )).rejects.toThrow("privileged custodied calibration"); expect(sameRouterProvider.dispatches).toBe(0); expect(onlyRemainingProvider.dispatches).toBe(0); }); - it("attributes a budget rejection without dispatching the judge", async () => { + it("fails closed when an LLM-contributing stage has no exposure ledger", async () => { + const candidate = observation(trace.id); + delete candidate.metadata.router.providerExposures; + const provider = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"must not dispatch"}', + ); + await expect(judgeFixedTraceObservation(trace, candidate, config(provider))) + .resolves.toMatchObject({ + status: "skipped", + failureReason: "candidate_not_judgeable", + }); + expect(provider.dispatches).toBe(0); + }); + + it("fails closed when a terminal or exposure provider identity is unknown or unledgered", async () => { + const terminalMismatch = observation(trace.id); + terminalMismatch.metadata.router.returnedProvider = "openai"; + terminalMismatch.metadata.router.returnedModel = "openai-hidden-fallback"; + const unknownExposure = observation(trace.id) as any; + unknownExposure.metadata.generation.providerExposures[0].returnedProvider = "unknown"; + const provider = new ScriptedJudgeProvider( + "google", + '{"pass":true,"score":4,"reason":"correct","finding":"must not dispatch"}', + ); + for (const candidate of [terminalMismatch, unknownExposure]) { + await expect(judgeFixedTraceObservation(trace, candidate, config(provider))) + .resolves.toMatchObject({ + status: "skipped", + failureReason: "candidate_not_judgeable", + }); + } + expect(provider.dispatches).toBe(0); + }); + + it("blocks a budgeted judge before any provider exposure without calibration", async () => { const delegate = new ScriptedJudgeProvider( "openai", '{"pass":true,"score":4,"reason":"correct"}', @@ -451,15 +512,14 @@ describe("fixed-trace independent judge", () => { config(provider), ); expect(result).toMatchObject({ - status: "not_dispatched_budget", - failureReason: "judge_budget_rejected", + status: "skipped", + failureReason: "judge_calibration_not_admitted", metadata: { usageKnown: false, estimatedCostUsd: 0 }, }); - expect(result.metadata.providerRequestSha256).toMatch(/^[a-f0-9]{64}$/); expect(delegate.dispatches).toBe(0); }); - it("requires and summarizes two distinct non-candidate judge providers", async () => { + it("blocks an otherwise independent panel without custodied calibration", async () => { const candidate = observation(trace.id); const openai = new ScriptedJudgeProvider( "openai", @@ -469,24 +529,21 @@ describe("fixed-trace independent judge", () => { "google", '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}', ); - const judgments = await runIndependentFixedTraceJudges( - [trace], - [candidate], - [config(openai), config(google)], - ); - expect(judgments).toHaveLength(FIXED_TRACE_MIN_INDEPENDENT_JUDGES); + await expect(runIndependentFixedTraceJudges( + [trace], [candidate], [config(openai), config(google)], + )).rejects.toThrow("privileged custodied calibration"); expect( - summarizeFixedTraceJudges([trace], [candidate], judgments), + summarizeFixedTraceJudges([trace], [candidate], []), ).toMatchObject({ expectedCases: 1, expectedJudgments: 2, - observedJudgments: 2, - judgedJudgments: 2, - complete: true, - judgmentCoverageRate: 1, - consensusPassRate: 1, - disagreementRate: 0, - comparisonEligible: true, + observedJudgments: 0, + judgedJudgments: 0, + complete: false, + judgmentCoverageRate: 0, + consensusPassRate: null, + disagreementRate: null, + comparisonEligible: false, }); }); @@ -501,11 +558,11 @@ describe("fixed-trace independent judge", () => { [observation(trace.id)], [config(openai)], ), - ).rejects.toThrow("requires at least two independent judge providers"); + ).rejects.toThrow("privileged custodied calibration"); expect(openai.dispatches).toBe(0); }); - it("records disagreement as a failed consensus without hiding completed coverage", async () => { + it("does not score disagreement without custodied calibration", async () => { const candidate = observation(trace.id); const openai = new ScriptedJudgeProvider( "openai", @@ -515,18 +572,16 @@ describe("fixed-trace independent judge", () => { "google", '{"pass":false,"score":2,"reason":"incomplete","finding":"The answer omits a required criterion."}', ); - const judgments = await runIndependentFixedTraceJudges( - [trace], - [candidate], - [config(openai), config(google)], - ); + await expect(runIndependentFixedTraceJudges( + [trace], [candidate], [config(openai), config(google)], + )).rejects.toThrow("privileged custodied calibration"); expect( - summarizeFixedTraceJudges([trace], [candidate], judgments), + summarizeFixedTraceJudges([trace], [candidate], []), ).toMatchObject({ - judgmentCoverageRate: 1, - consensusPassRate: 0, - disagreementRate: 1, - comparisonEligible: true, + judgmentCoverageRate: 0, + consensusPassRate: null, + disagreementRate: null, + comparisonEligible: false, }); }); }); diff --git a/server/tests/unit/addie/fixed-trace-runner.test.ts b/server/tests/unit/addie/fixed-trace-runner.test.ts index 9d9144de6f..c11d750f12 100644 --- a/server/tests/unit/addie/fixed-trace-runner.test.ts +++ b/server/tests/unit/addie/fixed-trace-runner.test.ts @@ -481,6 +481,12 @@ describe('fixed trace artifact runner', () => { expect(ambiguous.terminalStage).toBe('generation'); expect(ambiguousRouter.respondCalls).toHaveLength(1); expect(ambiguousGeneration.respondCalls).toHaveLength(1); + expect(ambiguous.metadata.router.providerExposures).toEqual([ + expect.objectContaining({ attempt: 1, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + ]); + expect(ambiguous.metadata.generation.providerExposures).toEqual([ + expect.objectContaining({ attempt: 1, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + ]); }); it('fails hybrid admission safe for tool-bearing, admin, thread, and unknown-privacy cases', () => { @@ -755,6 +761,10 @@ describe('fixed trace artifact runner', () => { estimatedCostUsd: 0.00007, }); expect(observation.metadata.generation.providerRequestSha256).toMatch(/^[a-f0-9]{64}$/); + expect(observation.metadata.generation.providerExposures).toEqual([ + expect.objectContaining({ attempt: 1, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + expect.objectContaining({ attempt: 2, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + ]); expect(generation.respondCalls).toHaveLength(2); expect(generation.respondCalls[0].toolChoice).toEqual({ type: 'tool', name: 'search_docs' }); expect(generation.respondCalls[1].toolChoice).toBeUndefined();