diff --git a/server/src/addie/eval/fixed-trace-architecture-diagnostic-execution.ts b/server/src/addie/eval/fixed-trace-architecture-diagnostic-execution.ts new file mode 100644 index 0000000000..26c78b56f2 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-architecture-diagnostic-execution.ts @@ -0,0 +1,627 @@ +/** + * Bounded paid execution for exactly one synthetic architecture diagnostic + * cell. This is diagnostic evidence only: it is not an external-final, + * production, canary, or comparison-eligibility path. + */ +import { createHash } from 'node:crypto'; +import { closeSync, fsyncSync, openSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import type { ModelProvider } from '../model-providers/model-provider.js'; +import { + FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_PACK_DIGEST, + FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_SUITE, + fixedTraceArchitectureDiagnosticPilotStageControls, +} from './fixed-trace-architecture-diagnostic.js'; +import { fixedTraceArchitectureArm, fixedTraceCommonToolDefinitions, fixedTraceHybridPolicy } from './fixed-trace-architecture.js'; +import { + BudgetedFixedTraceProvider, + FixedTraceBudget, + claimFixedTraceBudgetDiagnosticLease, + fixedTraceResponsePricingPolicy, + isTrustedBudgetedFixedTraceProvider, +} from './fixed-trace-budget.js'; +import { assertFixedTraceDiagnosticBudgetReconciliation } from './fixed-trace-diagnostic-run.js'; +import { datedPricingProfileIdentity, datedPricingProfilesForFixedTrace, datedPricingReservationCostUsd } from './dated-pricing-cohort.js'; +import { + FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_MAX_PREPARED_REQUEST_BYTES, + preflightFixedTraceRunnerConfig, + runFixedTraceArchitectureDiagnosticSonnetFullPack, + type FixedTraceProviderStageConfig, + type FixedTraceRunnerConfig, +} from './fixed-trace-runner.js'; +import { fixedTraceSuiteSha256, summarizeFixedTraceRun } from './fixed-trace-suite.js'; + +export const FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_EXECUTION_CELL = + 'architecture_diagnostic:anthropic:claude-haiku-4-5:claude-sonnet-5' as const; + +const EXECUTION_ARMS = Object.freeze([ + 'direct_generation', + 'two_stage_llm_router', + 'deterministic_policy_llm_fallback_hybrid', +] as const); + +type ExecutionArm = (typeof EXECUTION_ARMS)[number]; + +function sha256(content: string): string { + return createHash('sha256').update(content, 'utf8').digest('hex'); +} + +function freeze(value: T): T { + if (value && typeof value === 'object' && !Object.isFrozen(value)) { + for (const nested of Object.values(value as Record)) freeze(nested); + Object.freeze(value); + } + return value; +} + +function stage(provider: ModelProvider, kind: 'router' | 'generation'): FixedTraceProviderStageConfig { + const controls = fixedTraceArchitectureDiagnosticPilotStageControls()[kind]; + return Object.freeze({ ...controls, provider }); +} + +function requireStageProvider( + stageConfig: FixedTraceProviderStageConfig, + budget: FixedTraceBudget, +): void { + const policy = fixedTraceResponsePricingPolicy( + stageConfig.provider.id, + stageConfig.model, + stageConfig.pricing, + ); + if (!isTrustedBudgetedFixedTraceProvider(stageConfig.provider, budget, stageConfig.pricing, policy)) { + throw new Error('Fixed trace architecture diagnostic requires authenticated budgeted Anthropic stages'); + } +} + +export interface FixedTraceArchitectureDiagnosticCostCeiling { + readonly cell: typeof FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_EXECUTION_CELL; + readonly preparedRequestBytes: number; + readonly routerMaxDispatches: 40; + readonly generationMaxDispatches: 128; + readonly totalMaxDispatches: 168; + readonly routerReservationUsd: number; + readonly generationReservationUsd: number; + readonly totalUsd: number; + readonly requiredSoftMaxUsd: number; + readonly pricingProfileSha256: Readonly<{ router: string; generation: string }>; +} + +/** Static full-cell ceiling for all three exact architecture arms. */ +export function fixedTraceArchitectureDiagnosticCostCeiling(): FixedTraceArchitectureDiagnosticCostCeiling { + const controls = fixedTraceArchitectureDiagnosticPilotStageControls(); + const router = datedPricingProfilesForFixedTrace().find((entry) => entry.profileId === controls.router.pricing.profileId); + const generation = datedPricingProfilesForFixedTrace().find((entry) => entry.profileId === controls.generation.pricing.profileId); + if (!router || !generation) throw new Error('Fixed trace architecture diagnostic pricing is unavailable'); + fixedTraceResponsePricingPolicy('anthropic', controls.router.model, router); + fixedTraceResponsePricingPolicy('anthropic', controls.generation.model, generation); + const routerReservationUsd = 40 * datedPricingReservationCostUsd( + router, + FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_MAX_PREPARED_REQUEST_BYTES, + controls.router.maxOutputTokens, + ); + const generationReservationUsd = 128 * datedPricingReservationCostUsd( + generation, + FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_MAX_PREPARED_REQUEST_BYTES, + controls.generation.maxOutputTokens, + ); + return freeze({ + cell: FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_EXECUTION_CELL, + preparedRequestBytes: FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_MAX_PREPARED_REQUEST_BYTES, + routerMaxDispatches: 40, + generationMaxDispatches: 128, + totalMaxDispatches: 168, + routerReservationUsd, + generationReservationUsd, + totalUsd: routerReservationUsd + generationReservationUsd, + requiredSoftMaxUsd: routerReservationUsd + generationReservationUsd, + pricingProfileSha256: { + router: datedPricingProfileIdentity(router).digest, + generation: datedPricingProfileIdentity(generation).digest, + }, + }); +} + +export function fixedTraceArchitectureDiagnosticSourceBundle(files: readonly string[]): { + readonly files: readonly string[]; + readonly sha256: string; +} { + const ordered = [...new Set(files)].sort(); + if (ordered.length === 0 || ordered.some((file) => !file || file.startsWith('/') || file.includes('..'))) { + throw new Error('Fixed trace architecture diagnostic source bundle is invalid'); + } + const hash = createHash('sha256'); + for (const file of ordered) hash.update(file, 'utf8').update('\0').update(readFileSync(file)).update('\0'); + return freeze({ files: Object.freeze(ordered), sha256: hash.digest('hex') }); +} + +export function fixedTraceArchitectureDiagnosticPlan(input: Readonly<{ + sourceFiles: readonly string[]; + sourceBundleSha256: string; + promptConfigVersion: string; +}>) { + const controls = fixedTraceArchitectureDiagnosticPilotStageControls(); + return freeze({ + cell: FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_EXECUTION_CELL, + architectureDiagnosticMode: 'synthetic_sonnet_full_pack_v1', + packDigest: FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_PACK_DIGEST, + traceCount: 24, + arms: EXECUTION_ARMS, + router: { ...controls.router, providerId: controls.router.providerId }, + generation: { ...controls.generation, providerId: controls.generation.providerId }, + sourceFiles: [...input.sourceFiles], + sourceBundleSha256: input.sourceBundleSha256, + promptConfigVersion: input.promptConfigVersion, + wholeCellCostCeiling: fixedTraceArchitectureDiagnosticCostCeiling(), + diagnosticOnly: true, + comparisonEligible: false, + formalExternalFinal: 'unavailable', + }); +} + +export function fixedTraceArchitectureDiagnosticConfig(input: Readonly<{ + runId: string; + sourceBundleSha256: string; + gitCommit: string; + promptConfigVersion: string; + architectureArm: ExecutionArm; + router: ModelProvider; + generation: ModelProvider; +}>): FixedTraceRunnerConfig { + // Providers are live capabilities, not serializable evaluator evidence; + // freezing a supplied adapter's internals here would be surprising and is + // not the trust boundary. Admission instead requires immutable budgeted + // wrappers, while the runner snapshots the serializable config. + return Object.freeze({ + runId: input.runId, + sourceBundleSha256: input.sourceBundleSha256, + gitCommit: input.gitCommit, + gitDirty: false, + promptConfigVersion: input.promptConfigVersion, + traceSuite: FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_SUITE, + traceSuiteSha256: fixedTraceSuiteSha256(FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_SUITE), + toolDefinitions: fixedTraceCommonToolDefinitions(input.architectureArm), + toolDefinitionProvenance: 'evaluator_owned_common_tool_universe', + architectureArm: input.architectureArm, + architectureDiagnosticMode: 'synthetic_sonnet_full_pack_v1', + ...(input.architectureArm === 'deterministic_policy_llm_fallback_hybrid' + ? { hybridPolicy: fixedTraceHybridPolicy() } + : {}), + router: stage(input.router, 'router'), + generation: stage(input.generation, 'generation'), + }); +} + +export interface FixedTraceArchitectureDiagnosticAdmission { + /** Abandon an admitted cell before it begins; an admission cannot be reused. */ + release(): void; +} + +interface FixedTraceArchitectureDiagnosticAdmissionRecord { + readonly configs: readonly FixedTraceRunnerConfig[]; + readonly budget: FixedTraceBudget; + readonly lease: ReturnType; + readonly runRootId: string; + readonly runStartedAt: string; + readonly plan: ReturnType; + readonly budgetSoftMaxUsd: number; + state: 'admitted' | 'running' | 'released' | 'finished'; +} + +// The public admission is deliberately only a capability handle. Its mutable +// execution authority, providers, lease, budget, and provenance are held in +// this module-private registry so a matching-looking object cannot dispatch. +const admissions = new WeakMap(); + +function snapshotAdmissionPlan( + plan: ReturnType, + sourceBundleSha256: string, + promptConfigVersion: string, +): ReturnType { + const sourceFiles = plan?.sourceFiles; + if (!Array.isArray(sourceFiles) || sourceFiles.length === 0 || sourceFiles.some((file) => ( + typeof file !== 'string' || !file || file.startsWith('/') || file.includes('..') + ))) { + throw new Error('Fixed trace architecture diagnostic plan source files are invalid'); + } + const canonical = fixedTraceArchitectureDiagnosticPlan({ + sourceFiles: [...sourceFiles], sourceBundleSha256, promptConfigVersion, + }); + if (!isDeepStrictEqual(plan, canonical)) { + throw new Error('Fixed trace architecture diagnostic plan does not match its admitted provenance'); + } + return canonical; +} + +function releaseAdmission(record: FixedTraceArchitectureDiagnosticAdmissionRecord): void { + if (record.state === 'released' || record.state === 'finished') return; + record.lease.releaseWholeRunReservation(); + record.state = 'released'; +} + +function rejectAdmission(record: FixedTraceArchitectureDiagnosticAdmissionRecord, message: string): never { + // A malformed execution request is never retried with the same paid-cell + // authority. Release its unspent escrow before surfacing the rejection. + releaseAdmission(record); + throw new Error(message); +} + +function authenticatedAdmission( + admission: FixedTraceArchitectureDiagnosticAdmission, + input: Readonly<{ + runRootId: string; + runStartedAt: string; + plan: ReturnType; + budget: FixedTraceBudget; + }>, +): FixedTraceArchitectureDiagnosticAdmissionRecord { + const record = admissions.get(admission); + if (!record || !Object.isFrozen(admission)) { + throw new Error('Fixed trace architecture diagnostic admission is not authenticated'); + } + if (record.state !== 'admitted') { + throw new Error('Fixed trace architecture diagnostic admission is no longer available'); + } + if ( + input.runRootId !== record.runRootId + || input.runStartedAt !== record.runStartedAt + || input.budget !== record.budget + || !isDeepStrictEqual(input.plan, record.plan) + ) { + return rejectAdmission(record, 'Fixed trace architecture diagnostic artifact provenance does not match its admission'); + } + if (record.budget.softMaxUsd !== record.budgetSoftMaxUsd) { + return rejectAdmission(record, 'Fixed trace architecture diagnostic admitted budget was modified'); + } + if (record.configs.length !== EXECUTION_ARMS.length) { + return rejectAdmission(record, 'Fixed trace architecture diagnostic admission has an invalid arm count'); + } + for (const [index, config] of record.configs.entries()) { + const arm = EXECUTION_ARMS[index]; + if ( + !arm + || config.runId !== `${record.runRootId}:${arm}` + || config.architectureArm !== arm + || config.sourceBundleSha256 !== record.plan.sourceBundleSha256 + || config.promptConfigVersion !== record.plan.promptConfigVersion + || config.architectureDiagnosticMode !== 'synthetic_sonnet_full_pack_v1' + || !config.router + ) return rejectAdmission(record, 'Fixed trace architecture diagnostic admitted config is invalid'); + try { + requireStageProvider(config.router, record.budget); + requireStageProvider(config.generation, record.budget); + preflightFixedTraceRunnerConfig(config); + } catch (error) { + return rejectAdmission(record, error instanceof Error ? error.message : String(error)); + } + } + return record; +} + +/** + * Authenticate the only declared cell and reserve the complete bounded + * three-arm run before an immutable selector or output path is consumed. + */ +export function admitFixedTraceArchitectureDiagnostic(input: Readonly<{ + runRootId: string; + runStartedAt: string; + sourceBundleSha256: string; + gitCommit: string; + promptConfigVersion: string; + plan: ReturnType; + router: ModelProvider; + generation: ModelProvider; + budget: FixedTraceBudget; +}>): FixedTraceArchitectureDiagnosticAdmission { + if (!input.runRootId.trim()) throw new Error('Fixed trace architecture diagnostic run root ID is required'); + if (!input.runStartedAt.trim()) throw new Error('Fixed trace architecture diagnostic run start time is required'); + if (!/^[a-f0-9]{64}$/.test(input.sourceBundleSha256)) { + throw new Error('Fixed trace architecture diagnostic source bundle digest is invalid'); + } + if (!input.gitCommit.trim() || !input.promptConfigVersion.trim()) { + throw new Error('Fixed trace architecture diagnostic provenance is required'); + } + const plan = snapshotAdmissionPlan(input.plan, input.sourceBundleSha256, input.promptConfigVersion); + const ceiling = fixedTraceArchitectureDiagnosticCostCeiling(); + if (input.budget.softMaxUsd < ceiling.requiredSoftMaxUsd) { + throw new RangeError('Fixed trace architecture diagnostic soft budget is below the required whole-cell ceiling'); + } + const requested = EXECUTION_ARMS.map((architectureArm) => fixedTraceArchitectureDiagnosticConfig({ + ...input, runId: `${input.runRootId}:${architectureArm}`, architectureArm, + })); + for (const config of requested) { + requireStageProvider(config.router!, input.budget); + requireStageProvider(config.generation, input.budget); + preflightFixedTraceRunnerConfig(config); + } + const lease = claimFixedTraceBudgetDiagnosticLease( + input.budget, + [input.router, input.generation], + undefined, + ceiling.requiredSoftMaxUsd, + ); + const configs = requested.map((config) => fixedTraceArchitectureDiagnosticConfig({ + ...input, + runId: config.runId, + architectureArm: config.architectureArm as ExecutionArm, + router: lease.providerFor(input.router), + generation: lease.providerFor(input.generation), + })); + try { + for (const config of configs) preflightFixedTraceRunnerConfig(config); + } catch (error) { + lease.releaseWholeRunReservation(); + throw error; + } + let admission: FixedTraceArchitectureDiagnosticAdmission; + const record: FixedTraceArchitectureDiagnosticAdmissionRecord = { + configs: Object.freeze(configs), + budget: input.budget, + lease, + runRootId: input.runRootId, + runStartedAt: input.runStartedAt, + plan, + budgetSoftMaxUsd: input.budget.softMaxUsd, + state: 'admitted', + }; + admission = Object.freeze({ release: () => releaseAdmission(record) }); + admissions.set(admission, record); + return admission; +} + +/** Durable one-use cell declaration. */ +export function consumeFixedTraceArchitectureDiagnosticSelector(path: string, input: Readonly<{ + sourceBundleSha256: string; + promptConfigVersion: string; +}>): void { + let descriptor: number; + try { descriptor = openSync(path, 'wx', 0o600); } + catch (error) { throw new Error(`Fixed trace architecture diagnostic selector was already consumed: ${error instanceof Error ? error.message : String(error)}`); } + try { + writeFileSync(descriptor, `${JSON.stringify({ + cell: FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_EXECUTION_CELL, + architectureDiagnosticMode: 'synthetic_sonnet_full_pack_v1', + traceSuiteSha256: fixedTraceSuiteSha256(FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_SUITE), + sourceBundleSha256: input.sourceBundleSha256, + promptConfigVersion: input.promptConfigVersion, + })}\n`, 'utf8'); + } finally { closeSync(descriptor); } +} + +export interface FixedTraceArchitectureDiagnosticOutputReservation { + finalize(artifact: unknown): string; +} + +/** + * A finalization attempt is terminal even when its checksum cannot be + * persisted. Callers must preserve the claimed paths and must not dispatch + * the cell again. The durability flags say exactly which terminal evidence + * made it to disk before the failure. + */ +export class FixedTraceArchitectureDiagnosticOutputFinalizationError extends Error { + constructor( + readonly artifactDurable: boolean, + readonly checksumDurable: boolean, + cause: unknown, + ) { + super('Fixed trace architecture diagnostic output finalization failed', { cause }); + this.name = 'FixedTraceArchitectureDiagnosticOutputFinalizationError'; + } +} + +/** Reserve artifact and checksum identities together; neither is overwritten. */ +export function reserveFixedTraceArchitectureDiagnosticOutput(path: string): FixedTraceArchitectureDiagnosticOutputReservation { + const output = resolve(path); + const checksum = `${output}.sha256`; + let artifactDescriptor: number; + let checksumDescriptor: number; + try { + artifactDescriptor = openSync(output, 'wx', 0o600); + try { checksumDescriptor = openSync(checksum, 'wx', 0o600); } + catch (error) { + // A checksum collision means this reservation never became usable. Roll + // back our just-created artifact claim so a corrected retry is safe. + try { closeSync(artifactDescriptor); } finally { unlinkSync(output); } + throw error; + } + } catch (error) { + throw new Error(`Cannot exclusively reserve fixed-trace architecture diagnostic output: ${error instanceof Error ? error.message : String(error)}`); + } + let finalizationAttempted = false; + return Object.freeze({ + finalize(artifact: unknown): string { + if (finalizationAttempted) throw new Error('Fixed trace architecture diagnostic output finalization was already attempted'); + finalizationAttempted = true; + let artifactDurable = false; + let checksumDurable = false; + let finalizationFailure: unknown; + let finalizationFailed = false; + let digest: string | null = null; + try { + const content = `${JSON.stringify(artifact, null, 2)}\n`; + digest = sha256(content); + writeFileSync(artifactDescriptor!, content, 'utf8'); + fsyncSync(artifactDescriptor!); + artifactDurable = true; + writeFileSync(checksumDescriptor!, `${digest} ${output}\n`, 'utf8'); + fsyncSync(checksumDescriptor!); + checksumDurable = true; + } catch (error) { + finalizationFailure = error; + finalizationFailed = true; + } + try { closeSync(artifactDescriptor!); } + catch (error) { + if (!finalizationFailed) finalizationFailure = error; + finalizationFailed = true; + } + try { closeSync(checksumDescriptor!); } + catch (error) { + if (!finalizationFailed) finalizationFailure = error; + finalizationFailed = true; + } + if (finalizationFailed) { + throw new FixedTraceArchitectureDiagnosticOutputFinalizationError( + artifactDurable, checksumDurable, finalizationFailure, + ); + } + return digest!; + }, + }); +} + +/** + * A completed run's artifact is the only terminal evidence eligible for its + * claimed output. In particular, do not attempt to replace it with a setup + * failure artifact if persistence fails: the selector has already been + * consumed and a provider may have been dispatched. + */ +export function finalizeCompletedFixedTraceArchitectureDiagnosticArtifact( + output: FixedTraceArchitectureDiagnosticOutputReservation, + artifact: unknown, +): string { + try { + return output.finalize(artifact); + } catch (error) { + const durability = error instanceof FixedTraceArchitectureDiagnosticOutputFinalizationError + ? `artifact durable=${error.artifactDurable}; checksum durable=${error.checksumDurable}` + : 'artifact and checksum durability are unknown'; + throw new Error( + `Fixed trace architecture diagnostic completed, but terminal artifact finalization failed (${durability}). The selector remains consumed; do not dispatch this cell again.`, + { cause: error }, + ); + } +} + +/** Execute all three declared arms and retain every returned observation. */ +export async function runFixedTraceArchitectureDiagnosticArtifact(input: Readonly<{ + admission: FixedTraceArchitectureDiagnosticAdmission; + runRootId: string; + runStartedAt: string; + plan: ReturnType; + budget: FixedTraceBudget; +}>) { + const admission = authenticatedAdmission(input.admission, input); + admission.state = 'running'; + const runs: Array & { + observations: readonly import('./fixed-trace-suite.js').FixedTraceObservation[]; + }> = []; + let executionFailure: string | null = null; + let reconciliationFailure: string | null = null; + try { + for (const config of admission.configs) { + const observations: import('./fixed-trace-suite.js').FixedTraceObservation[] = []; + try { + const completed = await runFixedTraceArchitectureDiagnosticSonnetFullPack(config, (observation) => { + observations.push(observation); + }); + if (completed.length !== observations.length || completed.some((observation, index) => observation !== observations[index])) { + throw new Error('Fixed trace architecture diagnostic observation retention is inconsistent'); + } + const summarized = summarizeFixedTraceRun(observations, FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_SUITE); + if (observations.length !== 24 || summarized.summary.comparisonEligible !== false) { + throw new Error('Fixed trace architecture diagnostic did not retain the complete diagnostic-only denominator'); + } + runs.push(freeze({ + architectureArm: fixedTraceArchitectureArm(config.architectureArm), + runId: config.runId, + observations, + ...summarized, + })); + } catch (error) { + executionFailure = error instanceof Error ? error.message : String(error); + runs.push(freeze({ + architectureArm: fixedTraceArchitectureArm(config.architectureArm), + runId: config.runId, + observations, + complete: false, + failure: executionFailure, + })); + break; + } + } + } finally { + releaseAdmission(admission); + admission.state = 'finished'; + } + const budget = admission.budget.snapshot(); + try { + // The runner retains invocation identity, continuation request hashes, and + // custom-tool ledgers in each observation. This reconciles those retained + // observations with the budget wrapper's per-dispatch actual-usage + // settlement, or explicitly preserved unknown exposure. + assertFixedTraceDiagnosticBudgetReconciliation( + budget, + runs as Array<{ observations: readonly { metadata: import('./fixed-trace-suite.js').FixedTraceRunMetadata; terminalStatus: string }[] }>, + ); + } catch (error) { + reconciliationFailure = error instanceof Error ? error.message : String(error); + } + return freeze({ + artifactVersion: 'fixed_trace_architecture_diagnostic_execution_v1', + runRootId: admission.runRootId, + runStartedAt: admission.runStartedAt, + runCompletedAt: new Date().toISOString(), + plan: admission.plan, + budget, + diagnosticOnly: true, + comparisonEligible: false, + productionEligible: false, + canaryEligible: false, + promotionEvidenceEligible: false, + promotionBlocker: 'trusted_evaluator_context_unavailable', + formalExternalFinal: 'unavailable', + // Denominator coverage remains visible on each run summary. A paid call + // with unknown exposure, however, is not a complete settled execution. + complete: executionFailure === null && reconciliationFailure === null && !budget.exposureUnknown + && runs.length === EXECUTION_ARMS.length && runs.every((run) => run.summary !== undefined && ( + run.summary as { complete: boolean } + ).complete), + // Keep the former headline field for consumers while retaining an + // independent reconciliation result when execution itself also failed. + failure: executionFailure ?? reconciliationFailure, + executionFailure, + reconciliationFailure, + runs, + }); +} + +/** + * Preserve a setup failure after an output path was claimed without allowing a + * caller to invent artifact provenance outside the authenticated admission. + */ +export function fixedTraceArchitectureDiagnosticFailureArtifact( + admission: FixedTraceArchitectureDiagnosticAdmission, + error: unknown, +) { + const record = admissions.get(admission); + if (!record || !Object.isFrozen(admission)) { + throw new Error('Fixed trace architecture diagnostic admission is not authenticated'); + } + if (record.state === 'running' || record.state === 'finished') { + throw new Error('Fixed trace architecture diagnostic admission cannot produce a setup failure artifact'); + } + releaseAdmission(record); + record.state = 'finished'; + const executionFailure = error instanceof Error ? error.message : String(error); + return freeze({ + artifactVersion: 'fixed_trace_architecture_diagnostic_execution_v1', + runRootId: record.runRootId, + runStartedAt: record.runStartedAt, + runCompletedAt: new Date().toISOString(), + plan: record.plan, + budget: record.budget.snapshot(), + diagnosticOnly: true, + comparisonEligible: false, + productionEligible: false, + canaryEligible: false, + promotionEvidenceEligible: false, + promotionBlocker: 'trusted_evaluator_context_unavailable', + formalExternalFinal: 'unavailable', + complete: false, + failure: executionFailure, + executionFailure, + reconciliationFailure: null, + runs: [], + }); +} diff --git a/server/src/addie/eval/fixed-trace-runner.ts b/server/src/addie/eval/fixed-trace-runner.ts index 4fd0088705..82af3afe32 100644 --- a/server/src/addie/eval/fixed-trace-runner.ts +++ b/server/src/addie/eval/fixed-trace-runner.ts @@ -140,6 +140,13 @@ export const FIXED_TRACE_DIRECT_MODEL_SCREEN_GOOGLE_THREE_TURN_MODE = 'direct_mo export const FIXED_TRACE_DIRECT_FULL_SUITE_COMPARISON_MODE = 'direct_full_suite_model_comparison_v1' as const; /** Bounds every direct-full-suite prepared request before provider dispatch. */ export const FIXED_TRACE_DIRECT_FULL_SUITE_MAX_PREPARED_REQUEST_BYTES = 262_144; +/** + * The paid architecture diagnostic uses the same conservative request ceiling + * as the separately reviewed direct full-suite evaluator. This is a dispatch + * guard as well as a reservation input: a later continuation cannot silently + * exceed the whole-cell escrow that was admitted before evidence paths open. + */ +export const FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_MAX_PREPARED_REQUEST_BYTES = 262_144; export type FixedTraceDirectModelScreenGenerationCellId = | 'generation:anthropic:claude-sonnet-5:provider_default' | 'generation:anthropic:claude-haiku-4-5:provider_default' @@ -1746,6 +1753,7 @@ async function executeRouter( trace: FixedTraceCase, config: FixedTraceProviderStageConfig, assertBeforeDispatch: () => void, + architectureDiagnosticMode: FixedTraceArchitectureDiagnosticMode | undefined, ): Promise<{ request: ModelRequest; response: ModelResponse | null; @@ -1786,6 +1794,11 @@ async function executeRouter( signal: controller.signal, beforeDispatch: (prepared) => { assertBeforeDispatch(); + if ( + architectureDiagnosticMode === 'synthetic_sonnet_full_pack_v1' + && Buffer.byteLength(JSON.stringify(prepared.providerRequest), 'utf8') + > FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_MAX_PREPARED_REQUEST_BYTES + ) throw new FixedTracePreparationError('router', new Error('Fixed trace architecture diagnostic prepared request exceeds its reviewed byte ceiling')); dispatched = true; dispatchedCalls++; invocations.push(prepared); @@ -1971,6 +1984,7 @@ export async function runFixedTraceCase( executionTrace, executionConfig.router ?? (() => { throw new Error('Fixed trace runner router is missing'); })(), assertBeforeDispatch, + executionConfig.architectureDiagnosticMode, ); const generationNotRun = notRunStageMetadata(executionTrace); if (!routed.plan || routed.status) { @@ -2097,6 +2111,11 @@ export async function runFixedTraceCase( && Buffer.byteLength(JSON.stringify(prepared.providerRequest), 'utf8') > FIXED_TRACE_DIRECT_FULL_SUITE_MAX_PREPARED_REQUEST_BYTES ) throw new FixedTracePreparationError('generation', new Error('Fixed trace direct full-suite prepared request exceeds its reviewed byte ceiling')); + if ( + executionConfig.architectureDiagnosticMode === 'synthetic_sonnet_full_pack_v1' + && Buffer.byteLength(JSON.stringify(prepared.providerRequest), 'utf8') + > FIXED_TRACE_ARCHITECTURE_DIAGNOSTIC_MAX_PREPARED_REQUEST_BYTES + ) throw new FixedTracePreparationError('generation', new Error('Fixed trace architecture diagnostic prepared request exceeds its reviewed byte ceiling')); dispatched = true; dispatchedCalls++; invocations.push(prepared); @@ -2227,6 +2246,7 @@ export async function runFixedTraceCase( export async function runFixedTraceSuite( config: FixedTraceRunnerConfig, + onObservation?: (observation: FixedTraceObservation) => void, ): Promise { const identity = executionIdentity(config); preflightFixtureRegistrations(config, identity); @@ -2238,7 +2258,9 @@ export async function runFixedTraceSuite( const observations: FixedTraceObservation[] = []; for (const trace of iterationPlan) { assertExecutionIdentity(config, identity); - observations.push(await runFixedTraceCase(trace, config, identity.toolSchemaSha256)); + const observation = await runFixedTraceCase(trace, config, identity.toolSchemaSha256); + observations.push(observation); + onObservation?.(observation); } assertExecutionIdentity(config, identity); return observations; @@ -2303,11 +2325,12 @@ export async function runFixedTraceArchitectureDiagnosticSuite( */ export async function runFixedTraceArchitectureDiagnosticSonnetFullPack( config: FixedTraceRunnerConfig, + onObservation?: (observation: FixedTraceObservation) => void, ): Promise { if (config.architectureDiagnosticMode !== 'synthetic_sonnet_full_pack_v1') { throw new Error('Fixed trace Sonnet full-pack diagnostic requires synthetic_sonnet_full_pack_v1 mode'); } - const observations = await runFixedTraceSuite(config); + const observations = await runFixedTraceSuite(config, onObservation); if (observations.length !== 24) { throw new Error('Fixed trace Sonnet full-pack diagnostic did not preserve the complete denominator'); } diff --git a/server/tests/manual/fixed-trace-architecture-diagnostic-eval.ts b/server/tests/manual/fixed-trace-architecture-diagnostic-eval.ts new file mode 100644 index 0000000000..922beaa560 --- /dev/null +++ b/server/tests/manual/fixed-trace-architecture-diagnostic-eval.ts @@ -0,0 +1,199 @@ +/** One immutable, paid, synthetic 24-case architecture diagnostic cell. */ +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { execFileSync } from 'node:child_process'; + +interface Arguments { + readonly validateOnly: boolean; + readonly execute: boolean; + readonly output: string | null; + readonly selector: string | null; +} + +function option(name: string): string | undefined { + return process.argv.slice(2).find((argument) => argument.startsWith(`${name}=`))?.slice(name.length + 1); +} + +function parseArguments(): Arguments { + const values = process.argv.slice(2); + for (const argument of values) { + if (argument === '--validate-only' || argument === '--execute' || argument.startsWith('--output=') || argument.startsWith('--selector=')) continue; + throw new Error(`Unsupported fixed-trace architecture diagnostic option: ${argument}`); + } + const validateOnly = values.includes('--validate-only'); + const execute = values.includes('--execute'); + if (validateOnly === execute) throw new Error('Specify exactly one of --validate-only or --execute'); + if (values.filter((value) => value === '--validate-only').length > 1 || values.filter((value) => value === '--execute').length > 1) { + throw new Error('Specify exactly one execution mode flag'); + } + for (const name of ['--output=', '--selector=']) { + if (values.filter((argument) => argument.startsWith(name)).length > 1) { + throw new Error(`At most one ${name.slice(2, -1)} value is permitted`); + } + } + const output = option('--output') ?? null; + const selector = option('--selector') ?? null; + if (validateOnly && (output !== null || selector !== null)) { + throw new Error('Validate-only accepts no selector or output path'); + } + if (execute && (!output?.trim() || !selector?.trim())) { + throw new Error('--execute requires --output and --selector'); + } + if (execute) { + const identities = [resolve(output!), resolve(selector!), resolve(`${output!}.sha256`)]; + if (new Set(identities).size !== identities.length) { + throw new Error('Selector, artifact, and checksum paths must be distinct'); + } + } + return { validateOnly, execute, output, selector }; +} + +const arguments_ = parseArguments(); +// No evaluator/provider module (and no SDK) is loaded until malformed input +// is rejected. Stdout remains a single machine-readable record. +process.env.LOG_LEVEL = 'silent'; +const execution = await import('../../src/addie/eval/fixed-trace-architecture-diagnostic-execution.js'); +const budgetModule = await import('../../src/addie/eval/fixed-trace-budget.js'); +const { AnthropicModelProvider } = arguments_.execute + ? await import('../../src/addie/model-providers/anthropic-provider.js') + : { AnthropicModelProvider: null }; + +const sources = execution.fixedTraceArchitectureDiagnosticSourceBundle([ + 'server/src/addie/eval/fixed-trace-architecture-diagnostic-execution.ts', + 'server/src/addie/eval/fixed-trace-architecture-diagnostic.ts', + 'server/src/addie/eval/fixed-trace-architecture.ts', + 'server/src/addie/eval/fixed-trace-budget.ts', + 'server/src/addie/eval/fixed-trace-runner.ts', + 'server/src/addie/eval/fixed-trace-suite.ts', + 'server/src/addie/eval/fixed-trace-tool-loop.ts', + 'server/src/addie/model-providers/anthropic-provider.ts', + 'server/tests/manual/fixed-trace-architecture-diagnostic-eval.ts', +]); +const promptConfigVersion = createHash('sha256').update(readFileSync('server/src/addie/prompts.ts')) + .update(readFileSync('server/src/addie/rules/index.ts')).digest('hex'); +const plan = execution.fixedTraceArchitectureDiagnosticPlan({ + sourceFiles: sources.files, + sourceBundleSha256: sources.sha256, + promptConfigVersion, +}); + +if (arguments_.validateOnly) { + console.log(JSON.stringify({ + validateOnly: true, + providerCalls: 0, + selectorConsumed: false, + outputWritten: false, + plan, + })); + process.exit(0); +} + +if (existsSync(arguments_.selector!) || existsSync(arguments_.output!) || existsSync(`${arguments_.output!}.sha256`)) { + throw new Error('Selector, artifact, or checksum already exists; this diagnostic cell is one-shot'); +} +if (execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim()) { + throw new Error('Git source drift: execute only from an exact clean reviewed head'); +} +const gitCommit = execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { encoding: 'utf8' }).trim(); +if (!/^[a-f0-9]{40}$/.test(gitCommit)) throw new Error('Git source identity is unavailable'); +const apiKey = process.env.ADDIE_ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY; +if (!apiKey || !AnthropicModelProvider) throw new Error('ADDIE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY is required'); + +const ceiling = execution.fixedTraceArchitectureDiagnosticCostCeiling(); +const budget = new budgetModule.FixedTraceBudget(ceiling.requiredSoftMaxUsd); +const rawProvider = new AnthropicModelProvider(apiKey, undefined, { transportMaxRetries: 0 }); +const controls = (await import('../../src/addie/eval/fixed-trace-architecture-diagnostic.js')) + .fixedTraceArchitectureDiagnosticPilotStageControls(); +const router = new budgetModule.BudgetedFixedTraceProvider( + rawProvider, + budget, + controls.router.pricing, + budgetModule.fixedTraceResponsePricingPolicy('anthropic', controls.router.model, controls.router.pricing), +); +const generation = new budgetModule.BudgetedFixedTraceProvider( + rawProvider, + budget, + controls.generation.pricing, + budgetModule.fixedTraceResponsePricingPolicy('anthropic', controls.generation.model, controls.generation.pricing), +); +const runRootId = `architecture-diagnostic-${Date.now()}`; +const runStartedAt = new Date().toISOString(); +const admission = execution.admitFixedTraceArchitectureDiagnostic({ + runRootId, + runStartedAt, + sourceBundleSha256: sources.sha256, + gitCommit, + promptConfigVersion, + plan, + router, + generation, + budget, +}); +let output: ReturnType | null = null; +let completedArtifact: Awaited> | null = null; +let terminalArtifactFinalized = false; +try { + // Both evidence identities are durably claimed before the first provider + // dispatch. A reserved empty file is intentionally retained on a crash. + output = execution.reserveFixedTraceArchitectureDiagnosticOutput(arguments_.output!); + execution.consumeFixedTraceArchitectureDiagnosticSelector(arguments_.selector!, { + sourceBundleSha256: sources.sha256, + promptConfigVersion, + }); + completedArtifact = await execution.runFixedTraceArchitectureDiagnosticArtifact({ + admission, + runRootId, + runStartedAt, + plan, + budget, + }); + const artifactSha256 = execution.finalizeCompletedFixedTraceArchitectureDiagnosticArtifact(output, completedArtifact); + terminalArtifactFinalized = true; + console.log(JSON.stringify({ + output: arguments_.output, + artifactSha256, + complete: completedArtifact.complete, + diagnosticOnly: true, + comparisonEligible: false, + })); + if (!completedArtifact.complete) { + process.exitCode = 1; + console.error(`Fixed trace architecture diagnostic did not complete with fully known exposure; preserved artifact ${arguments_.output} (${artifactSha256})`); + } +} catch (error) { + if (output === null) { + admission.release(); + throw error; + } + if (completedArtifact !== null && !terminalArtifactFinalized) { + // The completed artifact was the only correct terminal evidence. Its + // finalizer has already sealed the reservation, so a setup-failure + // replacement would overwrite or obscure paid execution evidence. + const finalizationDetail = error instanceof Error ? ` ${error.message}` : ''; + throw new Error( + `Fixed trace architecture diagnostic completed, but its terminal artifact could not be finalized at ${arguments_.output}; the claimed artifact and checksum paths were retained without replacement. The selector remains consumed and this cell must not be dispatched again.${finalizationDetail}`, + { cause: error }, + ); + } + if (completedArtifact !== null) throw error; + let failureArtifact: ReturnType; + try { + failureArtifact = execution.fixedTraceArchitectureDiagnosticFailureArtifact(admission, error); + } catch (failureArtifactError) { + throw new Error( + `Fixed trace architecture diagnostic failed after its output and selector were claimed; the selector remains consumed and this cell must not be dispatched again.`, + { cause: failureArtifactError }, + ); + } + let artifactSha256: string; + try { + artifactSha256 = output.finalize(failureArtifact); + } catch (finalizationError) { + throw new Error( + `Fixed trace architecture diagnostic failed and its failure artifact could not be finalized; the selector remains consumed and this cell must not be dispatched again.`, + { cause: finalizationError }, + ); + } + throw new Error(`Fixed trace architecture diagnostic failed; preserved artifact ${arguments_.output} (${artifactSha256})`, { cause: error }); +} diff --git a/server/tests/unit/addie/fixed-trace-architecture-diagnostic-execution-cli.test.ts b/server/tests/unit/addie/fixed-trace-architecture-diagnostic-execution-cli.test.ts new file mode 100644 index 0000000000..3ba36fa4da --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-architecture-diagnostic-execution-cli.test.ts @@ -0,0 +1,43 @@ +import { spawnSync } from 'node:child_process'; +import { realpathSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const root = process.cwd(); +const tsx = realpathSync(resolve(root, 'node_modules/.bin/tsx')); +const providerFreeEnv = { PATH: process.env.PATH ?? '', NODE_ENV: 'test' }; + +function run(...arguments_: string[]) { + return spawnSync(process.execPath, [tsx, 'server/tests/manual/fixed-trace-architecture-diagnostic-eval.ts', ...arguments_], { + cwd: root, encoding: 'utf8', env: providerFreeEnv, + }); +} + +describe('fixed-trace architecture diagnostic execution CLI', () => { + it('validates the exact one-cell plan with no credential, output, selector, or provider setup', () => { + const result = run('--validate-only'); + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + const lines = result.stdout.trim().split('\n'); + expect(lines).toHaveLength(1); + expect(JSON.parse(lines[0]!)).toMatchObject({ + validateOnly: true, providerCalls: 0, selectorConsumed: false, outputWritten: false, + plan: { + cell: 'architecture_diagnostic:anthropic:claude-haiku-4-5:claude-sonnet-5', + traceCount: 24, diagnosticOnly: true, comparisonEligible: false, formalExternalFinal: 'unavailable', + wholeCellCostCeiling: { totalMaxDispatches: 168 }, + }, + }); + }, 20_000); + + it.each([ + ['--validate-only', '--output=/tmp/forbidden.json'], + ['--validate-only', '--cell=forged'], + ['--execute'], + ['--validate-only', '--validate-only'], + ])('rejects runtime selection or malformed paths %j', (...arguments_: string[]) => { + const result = run(...arguments_); + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(''); + }); +}); diff --git a/server/tests/unit/addie/fixed-trace-architecture-diagnostic-execution.test.ts b/server/tests/unit/addie/fixed-trace-architecture-diagnostic-execution.test.ts new file mode 100644 index 0000000000..f8c085c420 --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-architecture-diagnostic-execution.test.ts @@ -0,0 +1,262 @@ +import { createHash } from 'node:crypto'; +import { existsSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + admitFixedTraceArchitectureDiagnostic, + consumeFixedTraceArchitectureDiagnosticSelector, + finalizeCompletedFixedTraceArchitectureDiagnosticArtifact, + fixedTraceArchitectureDiagnosticCostCeiling, + fixedTraceArchitectureDiagnosticPlan, + reserveFixedTraceArchitectureDiagnosticOutput, + runFixedTraceArchitectureDiagnosticArtifact, +} from '../../../src/addie/eval/fixed-trace-architecture-diagnostic-execution.js'; +import { fixedTraceArchitectureDiagnosticPilotStageControls } from '../../../src/addie/eval/fixed-trace-architecture-diagnostic.js'; +import { + BudgetedFixedTraceProvider, + FixedTraceBudget, + fixedTraceResponsePricingPolicy, +} from '../../../src/addie/eval/fixed-trace-budget.js'; +import type { + ModelProvider, + ModelProviderCapabilities, + ModelRequest, + ModelRespondOptions, + ModelResponse, + NormalizedModelEvent, + PreparedModelInvocation, +} from '../../../src/addie/model-providers/model-provider.js'; + +const CAPABILITIES: ModelProviderCapabilities = { + streaming: false, structuredOutput: true, reasoning: true, + reasoningEfforts: ['provider_default'], customTools: true, + providerWebSearch: false, imageInput: false, documentInput: false, +}; + +class ScriptedAnthropicProvider implements ModelProvider { + readonly id = 'anthropic' as const; + readonly capabilities = CAPABILITIES; + readonly calls: ModelRequest[] = []; + + constructor( + private readonly unknownGenerationModel = false, + private readonly failAfterCalls: number | null = null, + ) {} + + prepare(request: ModelRequest): PreparedModelInvocation { + if (this.failAfterCalls !== null && this.calls.length >= this.failAfterCalls) { + throw new Error('synthetic mid-arm preparation failure'); + } + return { + provider: this.id, + model: request.model, + capabilities: this.capabilities, + requestMetadata: request.requestMetadata, + providerRequest: structuredClone(request) as unknown as Record, + }; + } + + async *respond(request: ModelRequest, options: ModelRespondOptions = {}): AsyncIterable { + await options.beforeDispatch?.(this.prepare(request)); + this.calls.push(structuredClone(request)); + const router = request.requestMetadata?.purpose === 'fixed_trace_router'; + const response: ModelResponse = { + provider: this.id, + model: router ? 'claude-haiku-4-5' : this.unknownGenerationModel ? 'unapproved-model' : 'claude-sonnet-5', + id: `synthetic-${this.calls.length}`, + content: [{ type: 'text', text: router + ? JSON.stringify({ action: 'respond', tool_sets: [], confidence: 'high', requires_depth: false, reason: 'Synthetic route.' }) + : 'Synthetic diagnostic response.' }], + finishReason: 'stop', providerFinishReason: 'stop', usage: { inputTokens: 10, outputTokens: 5 }, + }; + yield { type: 'response_start', provider: this.id, model: response.model, id: response.id }; + yield { type: 'text_delta', index: 0, text: response.content[0]!.type === 'text' ? response.content[0].text : '' }; + yield { type: 'response_complete', response }; + } +} + +const RUN_STARTED_AT = '2026-09-07T00:00:00.000Z'; + +function admitted(unknownGenerationModel = false, failAfterCalls: number | null = null) { + const ceiling = fixedTraceArchitectureDiagnosticCostCeiling(); + const budget = new FixedTraceBudget(ceiling.requiredSoftMaxUsd); + const raw = new ScriptedAnthropicProvider(unknownGenerationModel, failAfterCalls); + const controls = fixedTraceArchitectureDiagnosticPilotStageControls(); + const router = new BudgetedFixedTraceProvider( + raw, budget, controls.router.pricing, + fixedTraceResponsePricingPolicy('anthropic', controls.router.model, controls.router.pricing), + ); + const generation = new BudgetedFixedTraceProvider( + raw, budget, controls.generation.pricing, + fixedTraceResponsePricingPolicy('anthropic', controls.generation.model, controls.generation.pricing), + ); + const admission = admitFixedTraceArchitectureDiagnostic({ + runRootId: 'architecture-test-root', runStartedAt: RUN_STARTED_AT, + sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', + promptConfigVersion: 'architecture-test-prompt', plan: plan(), router, generation, budget, + }); + return { admission, budget, raw }; +} + +function plan() { + return fixedTraceArchitectureDiagnosticPlan({ + sourceFiles: ['synthetic.ts'], sourceBundleSha256: 'a'.repeat(64), promptConfigVersion: 'architecture-test-prompt', + }); +} + +describe('fixed-trace architecture diagnostic execution', () => { + it('predeclares one bounded all-arm Anthropic cell and no external-final authority', () => { + const ceiling = fixedTraceArchitectureDiagnosticCostCeiling(); + expect(ceiling).toMatchObject({ + preparedRequestBytes: 262_144, + routerMaxDispatches: 40, + generationMaxDispatches: 128, + totalMaxDispatches: 168, + requiredSoftMaxUsd: expect.any(Number), + }); + expect(plan()).toMatchObject({ + architectureDiagnosticMode: 'synthetic_sonnet_full_pack_v1', + traceCount: 24, + arms: ['direct_generation', 'two_stage_llm_router', 'deterministic_policy_llm_fallback_hybrid'], + diagnosticOnly: true, + comparisonEligible: false, + formalExternalFinal: 'unavailable', + }); + }); + + it('retains all three arms with provider identities, usage, and tool/continuation observations', async () => { + const { admission, budget, raw } = admitted(); + const artifact = await runFixedTraceArchitectureDiagnosticArtifact({ + admission, budget, runRootId: 'architecture-test-root', runStartedAt: '2026-09-07T00:00:00.000Z', plan: plan(), + }); + expect(raw.calls).toHaveLength(104); + expect(artifact).toMatchObject({ + complete: true, diagnosticOnly: true, comparisonEligible: false, + promotionEvidenceEligible: false, formalExternalFinal: 'unavailable', failure: null, + }); + expect(artifact.runs).toHaveLength(3); + for (const run of artifact.runs as Array<{ observations: Array<{ metadata: { router: { providerExposures: unknown[]; usage: unknown }; generation: { providerExposures: unknown[]; usage: unknown } }; tools: unknown[]; rejectedToolCalls: unknown[] }> }>) { + expect(run.observations).toHaveLength(24); + for (const observation of run.observations) { + expect(Array.isArray(observation.tools)).toBe(true); + expect(Array.isArray(observation.rejectedToolCalls)).toBe(true); + expect(Array.isArray(observation.metadata.router.providerExposures)).toBe(true); + expect(Array.isArray(observation.metadata.generation.providerExposures)).toBe(true); + } + } + expect(budget.snapshot()).toMatchObject({ reservedUsd: 0, dispatchedCalls: 104, completedCalls: 104, exposureUnknown: false }); + }); + + it('preserves unknown provider exposure in a finalized diagnostic artifact instead of inventing cost certainty', async () => { + const directory = mkdtempSync(join(tmpdir(), 'architecture-diagnostic-')); + const output = join(directory, 'artifact.json'); + const { admission, budget } = admitted(true); + const artifact = await runFixedTraceArchitectureDiagnosticArtifact({ + admission, budget, runRootId: 'architecture-test-root', runStartedAt: '2026-09-07T00:00:00.000Z', plan: plan(), + }); + const reservation = reserveFixedTraceArchitectureDiagnosticOutput(output); + const digest = reservation.finalize(artifact); + // Denominator coverage remains retained, but unknown paid exposure cannot + // be represented as a completed settled execution. + expect(artifact).toMatchObject({ complete: false, comparisonEligible: false, failure: null }); + expect(artifact.runs).toHaveLength(3); + expect(budget.snapshot()).toMatchObject({ exposureUnknown: true, completedCalls: 0, dispatchedCalls: 1, reservedUsd: 0 }); + expect(readFileSync(`${output}.sha256`, 'utf8')).toBe(`${digest} ${output}\n`); + expect(createHash('sha256').update(readFileSync(output, 'utf8')).digest('hex')).toBe(digest); + }); + + it('does not overwrite or redispatch a completed cell when terminal artifact finalization fails', async () => { + const { admission, budget, raw } = admitted(); + const artifact = await runFixedTraceArchitectureDiagnosticArtifact({ + admission, budget, runRootId: 'architecture-test-root', runStartedAt: RUN_STARTED_AT, plan: plan(), + }); + const finalized: unknown[] = []; + const output = Object.freeze({ + finalize(value: unknown): string { + finalized.push(value); + throw new Error('synthetic checksum finalization failure'); + }, + }); + + expect(() => finalizeCompletedFixedTraceArchitectureDiagnosticArtifact(output, artifact)) + .toThrow('selector remains consumed; do not dispatch this cell again'); + expect(finalized).toHaveLength(1); + expect(finalized[0]).toBe(artifact); + expect(raw.calls).toHaveLength(104); + await expect(runFixedTraceArchitectureDiagnosticArtifact({ + admission, budget, runRootId: 'architecture-test-root', runStartedAt: RUN_STARTED_AT, plan: plan(), + })).rejects.toThrow('no longer available'); + expect(raw.calls).toHaveLength(104); + }); + + it('never overwrites an artifact, checksum, or selector identity', () => { + const directory = mkdtempSync(join(tmpdir(), 'architecture-diagnostic-')); + const output = join(directory, 'artifact.json'); + const selector = join(directory, 'cell.used'); + writeFileSync(output, 'existing'); + expect(() => reserveFixedTraceArchitectureDiagnosticOutput(output)).toThrow('Cannot exclusively reserve'); + expect(readFileSync(output, 'utf8')).toBe('existing'); + const checksumCollisionOutput = join(directory, 'checksum-collision.json'); + writeFileSync(`${checksumCollisionOutput}.sha256`, 'existing checksum'); + expect(() => reserveFixedTraceArchitectureDiagnosticOutput(checksumCollisionOutput)).toThrow('Cannot exclusively reserve'); + expect(existsSync(checksumCollisionOutput)).toBe(false); + // The failed checksum-only claim leaves no artifact tombstone, so a safe + // retry can reserve both names once the conflicting checksum is resolved. + unlinkSync(`${checksumCollisionOutput}.sha256`); + reserveFixedTraceArchitectureDiagnosticOutput(checksumCollisionOutput).finalize({ retried: true }); + expect(readFileSync(checksumCollisionOutput, 'utf8')).toContain('retried'); + const failedFinalizationOutput = join(directory, 'failed-finalization.json'); + const failedFinalization = reserveFixedTraceArchitectureDiagnosticOutput(failedFinalizationOutput); + const unserializable: { self?: unknown } = {}; + unserializable.self = unserializable; + expect(() => failedFinalization.finalize(unserializable)).toThrow('output finalization failed'); + expect(() => failedFinalization.finalize({ replacement: true })) + .toThrow('finalization was already attempted'); + expect(readFileSync(failedFinalizationOutput, 'utf8')).toBe(''); + expect(readFileSync(`${failedFinalizationOutput}.sha256`, 'utf8')).toBe(''); + consumeFixedTraceArchitectureDiagnosticSelector(selector, { + sourceBundleSha256: 'a'.repeat(64), promptConfigVersion: 'prompt', + }); + expect(() => consumeFixedTraceArchitectureDiagnosticSelector(selector, { + sourceBundleSha256: 'a'.repeat(64), promptConfigVersion: 'prompt', + })).toThrow('already consumed'); + }); + + it('rejects forged admissions and contradictory admitted provenance before any provider dispatch', async () => { + const { admission, budget, raw } = admitted(); + const forged = Object.freeze({ release() {} }); + await expect(runFixedTraceArchitectureDiagnosticArtifact({ + admission: forged as unknown as typeof admission, + budget, runRootId: 'architecture-test-root', runStartedAt: RUN_STARTED_AT, plan: plan(), + })).rejects.toThrow('not authenticated'); + await expect(runFixedTraceArchitectureDiagnosticArtifact({ + admission, + budget, + runRootId: 'architecture-test-root', + runStartedAt: RUN_STARTED_AT, + plan: fixedTraceArchitectureDiagnosticPlan({ + sourceFiles: ['synthetic.ts'], sourceBundleSha256: 'b'.repeat(64), promptConfigVersion: 'architecture-test-prompt', + }), + })).rejects.toThrow('provenance does not match'); + expect(raw.calls).toHaveLength(0); + admission.release(); + }); + + it('retains observations completed before a fatal mid-arm runner failure', async () => { + const { admission, budget, raw } = admitted(false, 4); + const artifact = await runFixedTraceArchitectureDiagnosticArtifact({ + admission, budget, runRootId: 'architecture-test-root', runStartedAt: RUN_STARTED_AT, plan: plan(), + }); + expect(raw.calls).toHaveLength(4); + expect(artifact).toMatchObject({ + complete: false, + failure: expect.stringContaining('generation request preparation failed'), + executionFailure: expect.stringContaining('generation request preparation failed'), + reconciliationFailure: null, + }); + expect(artifact.runs).toHaveLength(1); + expect((artifact.runs[0] as { observations: unknown[] }).observations).toHaveLength(4); + expect(budget.snapshot()).toMatchObject({ dispatchedCalls: 4, completedCalls: 4, reservedUsd: 0 }); + }); +});