From 8040493967256b6766c5b0cc5e23d052d37dfdfc Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 5 Sep 2026 23:01:23 +0000 Subject: [PATCH] fix(addie): make fixed-trace dispatch dormant --- .../addie/eval/fixed-trace-diagnostic-cli.ts | 36 +- .../addie/eval/fixed-trace-diagnostic-run.ts | 51 ++- .../eval/fixed-trace-evidence-prerequisite.ts | 5 +- server/src/addie/eval/fixed-trace-runner.ts | 94 ++++- server/src/addie/eval/fixed-trace-suite.ts | 15 +- .../src/addie/eval/fixed-trace-tool-loop.ts | 23 + .../tests/manual/fixed-trace-provider-eval.ts | 393 ++---------------- .../addie/fixed-trace-diagnostic-cli.test.ts | 114 +++-- .../fixed-trace-dormant-dispatch.test.ts | 40 ++ .../unit/addie/fixed-trace-judge.test.ts | 2 +- .../unit/addie/fixed-trace-rollout.test.ts | 1 + .../unit/addie/fixed-trace-runner.test.ts | 60 +++ .../unit/addie/fixed-trace-tool-loop.test.ts | 60 +++ 13 files changed, 413 insertions(+), 481 deletions(-) create mode 100644 server/tests/unit/addie/fixed-trace-dormant-dispatch.test.ts diff --git a/server/src/addie/eval/fixed-trace-diagnostic-cli.ts b/server/src/addie/eval/fixed-trace-diagnostic-cli.ts index ad6b7694a5..6a1387599a 100644 --- a/server/src/addie/eval/fixed-trace-diagnostic-cli.ts +++ b/server/src/addie/eval/fixed-trace-diagnostic-cli.ts @@ -1,38 +1,10 @@ export interface FixedTraceDiagnosticCliArguments { - providers?: string; - architectureArm?: string; - suite?: string; - softMaxUsd?: string; - output?: string; validateOnly: boolean; } -const NAMES = new Set(['providers', 'architecture-arm', 'suite', 'soft-max-usd', 'output', 'validate-only']); - -/** Strict, side-effect-free parser for the diagnostic-only manual evaluator. */ +/** The pinned planning artifact has no runtime controls or output surface. */ export function parseFixedTraceDiagnosticCliArguments(values: readonly string[]): FixedTraceDiagnosticCliArguments { - const seen = new Map(); - for (const value of values) { - if (value === '--judge-providers' || value.startsWith('--judge-providers=')) { - throw new Error('--judge-providers is unavailable_pending_trusted_coordinator'); - } - const bare = value.match(/^--([a-z][a-z-]*)$/); - const assigned = value.match(/^--([a-z][a-z-]*)=(.+)$/); - const name = bare?.[1] ?? assigned?.[1]; - if (!name || !NAMES.has(name)) throw new Error(`Unknown or malformed fixed-trace option: ${value}`); - if (seen.has(name)) throw new Error(`Duplicate fixed-trace option: --${name}`); - if (bare && name !== 'validate-only') throw new Error(`--${name} requires =value`); - if (assigned && name === 'validate-only' && assigned[2] !== 'true') { - throw new Error('--validate-only accepts only the bare flag or =true'); - } - seen.set(name, bare ? true : assigned![2]); - } - return { - providers: typeof seen.get('providers') === 'string' ? seen.get('providers') as string : undefined, - architectureArm: typeof seen.get('architecture-arm') === 'string' ? seen.get('architecture-arm') as string : undefined, - suite: typeof seen.get('suite') === 'string' ? seen.get('suite') as string : undefined, - softMaxUsd: typeof seen.get('soft-max-usd') === 'string' ? seen.get('soft-max-usd') as string : undefined, - output: typeof seen.get('output') === 'string' ? seen.get('output') as string : undefined, - validateOnly: seen.get('validate-only') === true || seen.get('validate-only') === 'true', - }; + if (values.length === 0) return { validateOnly: false }; + if (values.length === 1 && values[0] === '--validate-only') return { validateOnly: true }; + throw new Error('This planning-only evaluator accepts only the bare --validate-only flag'); } diff --git a/server/src/addie/eval/fixed-trace-diagnostic-run.ts b/server/src/addie/eval/fixed-trace-diagnostic-run.ts index e77b46df7d..f50ca91b4a 100644 --- a/server/src/addie/eval/fixed-trace-diagnostic-run.ts +++ b/server/src/addie/eval/fixed-trace-diagnostic-run.ts @@ -22,6 +22,8 @@ import { fixedTraceResponsePricingPolicy, isTrustedBudgetedFixedTraceProvider, } from './fixed-trace-budget.js'; +import { types } from 'node:util'; +import { snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js'; export interface FixedTraceDiagnosticProviderPlan { readonly name: string; @@ -63,6 +65,17 @@ function ownDataProperty(source: unknown, name: string, owner: string): unknown return descriptor.value; } +function assertClosedOwnDataRecord(source: unknown, fields: readonly string[], owner: string): void { + if (typeof source !== 'object' || source === null || types.isProxy(source) || Object.getPrototypeOf(source) !== Object.prototype) { + throw new Error(`Fixed trace diagnostic ${owner} must be a plain non-Proxy object`); + } + const keys = Reflect.ownKeys(source); + if (keys.length !== fields.length || keys.some((key) => typeof key !== 'string' || !fields.includes(key))) { + throw new Error(`Fixed trace diagnostic ${owner} must contain exactly its approved fields`); + } + for (const field of fields) ownDataProperty(source, field, owner); +} + const DIAGNOSTIC_PRICING_FIELDS = [ 'profileId', 'inputUsdPerMillionTokens', @@ -75,19 +88,7 @@ const DIAGNOSTIC_PRICING_FIELDS = [ ] as const; function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing { - const prototype = typeof pricing === 'object' && pricing !== null - ? Object.getPrototypeOf(pricing) - : null; - if ( - typeof pricing !== 'object' - || pricing === null - || (prototype !== Object.prototype && prototype !== null) - ) throw new Error(`Fixed trace diagnostic ${owner} must be a plain pricing object`); - const keys = Reflect.ownKeys(pricing); - if ( - keys.length !== DIAGNOSTIC_PRICING_FIELDS.length - || keys.some((key) => typeof key !== 'string' || !DIAGNOSTIC_PRICING_FIELDS.includes(key as typeof DIAGNOSTIC_PRICING_FIELDS[number])) - ) throw new Error(`Fixed trace diagnostic ${owner} must contain only approved pricing fields`); + assertClosedOwnDataRecord(pricing, DIAGNOSTIC_PRICING_FIELDS, owner); // Structured cloning calls nested getters. Copy each approved data // descriptor instead, so a price cannot change between validation and use. return Object.freeze({ @@ -105,6 +106,10 @@ function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing { function snapshotStageConfig(config: unknown, owner: string): FixedTraceProviderStageConfig { // Read each untrusted stage property exactly once. Later checks use only // this detached plain object, never a caller-controlled getter or proxy. + assertClosedOwnDataRecord(config, [ + 'provider', 'model', 'reasoningEffort', 'maxOutputTokens', 'timeoutMs', + 'maxIterations', 'transportRetries', 'samplingMode', 'temperature', 'pricing', + ], owner); const provider = ownDataProperty(config, 'provider', owner); const model = ownDataProperty(config, 'model', owner); const reasoningEffort = ownDataProperty(config, 'reasoningEffort', owner); @@ -132,12 +137,7 @@ function snapshotStageConfig(config: unknown, owner: string): FixedTraceProvider function snapshotBaseConfig( config: FixedTraceDiagnosticArtifactOptions['baseConfig'], ): FixedTraceDiagnosticArtifactOptions['baseConfig'] { - const { traceSuite, toolDefinitions, ...serializable } = config; - return Object.freeze({ - ...structuredClone(serializable), - traceSuite: deepFreeze(structuredClone(traceSuite)), - toolDefinitions: deepFreeze(structuredClone(toolDefinitions)), - }); + return snapshotFixedTraceJson(config, 'fixed trace diagnostic base config') as FixedTraceDiagnosticArtifactOptions['baseConfig']; } function snapshotPlans( @@ -147,6 +147,19 @@ function snapshotPlans( if (!Array.isArray(suppliedPlans) || suppliedPlans.length === 0) { throw new Error('Fixed trace diagnostic run requires one or more provider plans'); } + if (types.isProxy(suppliedPlans) || Object.getPrototypeOf(suppliedPlans) !== Array.prototype || Object.getOwnPropertySymbols(suppliedPlans).length !== 0) { + throw new Error('Fixed trace diagnostic provider plans must be a plain non-Proxy array'); + } + const planDescriptors = Object.getOwnPropertyDescriptors(suppliedPlans); + for (const key of Object.keys(planDescriptors)) { + if (key === 'length') continue; + if (!/^(0|[1-9][0-9]*)$/.test(key) || !('value' in planDescriptors[key]!) || !planDescriptors[key]!.enumerable) { + throw new Error('Fixed trace diagnostic provider plans contain an accessor or extra property'); + } + } + for (const [index, suppliedPlan] of suppliedPlans.entries()) { + assertClosedOwnDataRecord(suppliedPlan, ['name', 'router', 'generation'], `provider plan ${index}`); + } const plans = Object.freeze(suppliedPlans.map((suppliedPlan, index) => Object.freeze({ // Do not validate while reading: a plan accessor must not be able to // return one identity for validation and another for execution. diff --git a/server/src/addie/eval/fixed-trace-evidence-prerequisite.ts b/server/src/addie/eval/fixed-trace-evidence-prerequisite.ts index 66cc8bd5fc..00ab861bf4 100644 --- a/server/src/addie/eval/fixed-trace-evidence-prerequisite.ts +++ b/server/src/addie/eval/fixed-trace-evidence-prerequisite.ts @@ -34,6 +34,7 @@ type FixedTraceTerminalStatus = | "malformed" | "provider_error" | "timeout_after_dispatch" + | "unknown_exposure" | "not_dispatched_budget" | "not_admitted_architecture"; type FixedTraceInvocationStage = "router" | "generation" | "judge" | "simulator"; @@ -254,7 +255,7 @@ type FixedTraceExtraSchemaVersion = FixedTraceAssertTrue>; // @ts-expect-error terminal statuses cannot admit a member outside their closed domain -type FixedTraceExtraTerminalStatus = FixedTraceAssertTrue>; +type FixedTraceExtraTerminalStatus = FixedTraceAssertTrue>; // @ts-expect-error finish reasons cannot admit a member outside their closed domain type FixedTraceExtraFinishReason = FixedTraceAssertTrue>; // @ts-expect-error completeness outcomes cannot admit a member outside their closed domain @@ -310,7 +311,7 @@ export const FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS: }, timingAndOutcome: { preparedAt: { type: "utc_timestamp" }, dispatchedAt: { type: "nullable_utc_timestamp" }, completedAt: { type: "nullable_utc_timestamp" }, latencyMs: { type: "nullable_number" }, timeout: { type: "boolean" }, - errorCode: { type: "nullable_string" }, terminalStatus: fixedTraceEnum()(["complete", "ignored", "reacted", "refusal", "truncated", "empty", "malformed", "provider_error", "timeout_after_dispatch", "not_dispatched_budget", "not_admitted_architecture"]), + errorCode: { type: "nullable_string" }, terminalStatus: fixedTraceEnum()(["complete", "ignored", "reacted", "refusal", "truncated", "empty", "malformed", "provider_error", "timeout_after_dispatch", "unknown_exposure", "not_dispatched_budget", "not_admitted_architecture"]), finishReason: fixedTraceNullableEnum()(["stop", "tool_calls", "length", "refusal", "continue"]), outputSha256: { type: "nullable_sha256" }, }, usageAndPricing: { diff --git a/server/src/addie/eval/fixed-trace-runner.ts b/server/src/addie/eval/fixed-trace-runner.ts index 63eee1e5ac..37a3963337 100644 --- a/server/src/addie/eval/fixed-trace-runner.ts +++ b/server/src/addie/eval/fixed-trace-runner.ts @@ -182,10 +182,6 @@ function fixedTraceCommonToolEnvironment(): FixedTraceCommonToolEnvironmentBindi }); } -/** - * An evaluator-owned execution contract changed after its request snapshot was - * made. This is neither a provider failure nor a scored terminal outcome. - */ class FixedTraceExecutionIdentityError extends Error { constructor(message: string) { super(message); @@ -407,6 +403,43 @@ interface StageInvocationState { latencyMs: number; } +function providerExposures( + state: StageInvocationState, + response?: ModelResponse, + recordedExposures?: NonNullable, +): FixedTraceModelStageMetadata["providerExposures"] { + if (recordedExposures) return deepFreeze(recordedExposures.map((exposure) => ({ ...exposure }))); + return deepFreeze( + state.invocations.map((prepared, index) => ({ + attempt: index + 1, + preparedProvider: prepared.provider, + preparedModel: prepared.model, + returnedProvider: + response && index === state.invocations.length - 1 ? response.provider : null, + returnedModel: + response && index === state.invocations.length - 1 ? response.model : null, + })), + ); +} + +function hasCompleteReturnedProviderIdentities( + state: StageInvocationState, + response?: ModelResponse, + recordedExposures?: NonNullable, +): boolean { + const exposures = providerExposures(state, response, recordedExposures) ?? []; + return exposures.length === state.dispatchedCalls && exposures.every( + (exposure, index) => { + const prepared = state.invocations[index]; + return exposure.attempt === index + 1 && + prepared !== undefined && + exposure.preparedProvider === prepared.provider && + exposure.preparedModel === prepared.model && + Boolean(exposure.returnedProvider && exposure.returnedModel); + }, + ); +} + function canonicalJson(value: unknown): string { if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); if (typeof value === 'number') { @@ -534,6 +567,7 @@ function providerStageMetadata( response: ModelResponse, usage: ModelUsage, state: StageInvocationState, + recordedExposures?: NonNullable, ): FixedTraceModelStageMetadata { // Provider responses are outside evaluator ownership. Retaining their usage // object would let a later provider turn mutate already-recorded cost and @@ -548,6 +582,7 @@ function providerStageMetadata( requestedModel: config.model, returnedProvider: response.provider, returnedModel: response.model, + providerExposures: providerExposures(state, response, recordedExposures), modelResolution: modelResolution(config, response), promptSha256: promptSha256(request), providerRequestSha256: providerRequestSha256(state.invocations), @@ -575,6 +610,7 @@ function localStageMetadata( config: FixedTraceProviderStageConfig, state: StageInvocationState, usage?: ModelUsage, + recordedExposures?: NonNullable, ): FixedTraceModelStageMetadata { const recordedUsage = usage === undefined ? undefined : deepFreeze(structuredClone(usage)); return { @@ -585,6 +621,7 @@ function localStageMetadata( requestedModel: config.model, returnedProvider: null, returnedModel: null, + providerExposures: providerExposures(state, undefined, recordedExposures), modelResolution: 'local', promptSha256: promptSha256(request), providerRequestSha256: providerRequestSha256(state.invocations), @@ -615,6 +652,7 @@ function notRunStageMetadata(trace: FixedTraceCase): FixedTraceModelStageMetadat requestedModel: null, returnedProvider: null, returnedModel: null, + providerExposures: Object.freeze([]), modelResolution: null, promptSha256: null, providerRequestSha256: null, @@ -986,7 +1024,9 @@ async function executeRouter( const state = { invocations, dispatched, dispatchedCalls, latencyMs: Date.now() - startedAt }; const metadata = providerStageMetadata(request, config, response, response.usage, state); const output = extractRouterResponseText(response.content); - const status = terminalStatusForFinishReason(response.finishReason, output); + const status = hasCompleteReturnedProviderIdentities(state, response) + ? terminalStatusForFinishReason(response.finishReason, output) + : 'unknown_exposure'; if (status !== 'complete') return { request, response, plan: null, output, status, metadata }; try { return { @@ -1009,12 +1049,15 @@ async function executeRouter( : timedOut && dispatched ? 'timeout_after_dispatch' : 'provider_error'; + const terminalStatus = dispatched && !hasCompleteReturnedProviderIdentities(state) + ? 'unknown_exposure' + : status; return { request, response: null, plan: null, - output: fallbackOutput(status), - status, + output: fallbackOutput(terminalStatus), + status: terminalStatus, metadata: localStageMetadata(request, config, state), }; } finally { @@ -1058,7 +1101,7 @@ export async function runFixedTraceCase( if (architectureArm.id === 'direct_generation') { // The evaluator's receipts and fixture facts are diagnostic only; an // admission result can never open a direct-production dispatch path. - return { + return { traceId: executionTrace.id, metadata: baseMetadata( executionTrace, @@ -1077,8 +1120,8 @@ export async function runFixedTraceCase( route: null, tools: [], rejectedToolCalls: [], - }; - } + }; +} const hybridDecision = architectureArm.id === 'deterministic_policy_llm_fallback_hybrid' ? decideFixedTraceHybridRoute({ message: executionTrace.request.message, @@ -1233,8 +1276,15 @@ export async function runFixedTraceCase( result.response, result.usage, state, + result.providerExposures, ); - const terminalStatus = terminalStatusForFinishReason(result.response.finishReason, result.text); + const terminalStatus = hasCompleteReturnedProviderIdentities( + state, + undefined, + result.providerExposures, + ) && returnedModelUsesRecordedPricing(generationConfig, result.response) + ? terminalStatusForFinishReason(result.response.finishReason, result.text) + : 'unknown_exposure'; return { traceId: executionTrace.id, metadata: baseMetadata(executionTrace, executionConfig, toolSchemaSha256, routed.metadata, generation), @@ -1262,21 +1312,35 @@ export async function runFixedTraceCase( : timedOut && dispatched ? 'timeout_after_dispatch' : 'provider_error'; - const generation = localStageMetadata(generationRequest, generationConfig, { + const state = { invocations, dispatched, dispatchedCalls, latencyMs: Date.now() - startedAt, - }, checkpoint?.usage); + }; + const generation = localStageMetadata( + generationRequest, + generationConfig, + state, + checkpoint?.usage, + checkpoint?.providerExposures, + ); + const finalTerminalStatus = dispatched && !hasCompleteReturnedProviderIdentities( + state, + undefined, + checkpoint?.providerExposures, + ) + ? 'unknown_exposure' + : terminalStatus; return { traceId: executionTrace.id, metadata: baseMetadata(executionTrace, executionConfig, toolSchemaSha256, routed.metadata, generation), terminalStage: 'generation', - terminalStatus, + terminalStatus: finalTerminalStatus, boundaryReason: error instanceof FixedTraceToolLoopBoundaryError ? error.reason : null, localReplacementReason: null, finishReason: null, - output: fallbackOutput(terminalStatus), + output: fallbackOutput(finalTerminalStatus), flagged: true, route, tools: checkpoint ? [...checkpoint.tools] : [], diff --git a/server/src/addie/eval/fixed-trace-suite.ts b/server/src/addie/eval/fixed-trace-suite.ts index 8ea8e3676c..c19317d0a1 100644 --- a/server/src/addie/eval/fixed-trace-suite.ts +++ b/server/src/addie/eval/fixed-trace-suite.ts @@ -78,6 +78,8 @@ export type FixedTraceTerminalStatus = | 'malformed' | 'provider_error' | 'timeout_after_dispatch' + /** A dispatched attempt has no complete prepared/returned identity ledger. */ + | 'unknown_exposure' | 'not_dispatched_budget' /** A direct architecture candidate was rejected before provider dispatch. */ | 'not_admitted_architecture'; @@ -305,6 +307,14 @@ export interface FixedTraceModelStageMetadata { requestedModel: string | null; returnedProvider: ModelProviderId | null; returnedModel: string | null; + /** Identity-only ledger of every prepared attempt; never contains payloads. */ + providerExposures?: readonly { + attempt: number; + preparedProvider: ModelProviderId; + preparedModel: string; + returnedProvider: ModelProviderId | null; + returnedModel: string | null; + }[]; modelResolution: 'exact' | 'provider_canonicalized' | 'local' | null; promptSha256: string | null; providerRequestSha256: string | null; @@ -3054,7 +3064,7 @@ export function gradeFixedTrace( if (observation.terminalStatus === 'refusal' && observation.finishReason !== 'refusal') failures.push('finish_reason_mismatch'); const failureStatuses: ReadonlyArray = [ - 'malformed', 'provider_error', 'timeout_after_dispatch', 'not_dispatched_budget', 'not_admitted_architecture', + 'malformed', 'provider_error', 'timeout_after_dispatch', 'unknown_exposure', 'not_dispatched_budget', 'not_admitted_architecture', ]; if (observation.terminalStage === 'admission') { if ( @@ -3099,7 +3109,7 @@ export function gradeFixedTrace( ) failures.push('generation_stage_mismatch'); const metadataPass = provenanceFailures.length === 0; - const terminalFailure = ['refusal', 'truncated', 'empty', 'malformed', 'provider_error', 'timeout_after_dispatch', 'not_dispatched_budget', 'not_admitted_architecture'] + const terminalFailure = ['refusal', 'truncated', 'empty', 'malformed', 'provider_error', 'timeout_after_dispatch', 'unknown_exposure', 'not_dispatched_budget', 'not_admitted_architecture'] .includes(observation.terminalStatus); return { traceId: trace.id, @@ -3271,6 +3281,7 @@ export function summarizeFixedTraceRun( empty: 0, malformed: 0, provider_error: 0, + unknown_exposure: 0, timeout_after_dispatch: 0, not_dispatched_budget: 0, not_admitted_architecture: 0, diff --git a/server/src/addie/eval/fixed-trace-tool-loop.ts b/server/src/addie/eval/fixed-trace-tool-loop.ts index 20c11fab88..ae5b2f7c0b 100644 --- a/server/src/addie/eval/fixed-trace-tool-loop.ts +++ b/server/src/addie/eval/fixed-trace-tool-loop.ts @@ -40,6 +40,7 @@ export interface FixedTraceToolLoopCheckpoint { usage: ModelUsage; tools: ReadonlyArray; rejectedToolCalls: ReadonlyArray; + providerExposures: ReadonlyArray; } export class FixedTraceToolLoopBoundaryError extends Error { @@ -56,6 +57,15 @@ export interface FixedTraceToolExecution extends FixedTraceToolObservation { sequence: number; } +/** Identity-only record for a dispatched model turn; never prompt data. */ +export interface FixedTraceProviderExposure { + attempt: number; + preparedProvider: PreparedModelInvocation['provider']; + preparedModel: string; + returnedProvider: ModelResponse['provider']; + returnedModel: string; +} + export interface FixedTraceToolLoopResult { response: ModelResponse; text: string; @@ -64,6 +74,7 @@ export interface FixedTraceToolLoopResult { usage: ModelUsage; tools: ReadonlyArray; invocations: ReadonlyArray; + providerExposures: ReadonlyArray; } export interface FixedTraceToolLoopOptions { @@ -291,6 +302,7 @@ export async function executeFixedTraceToolLoop( const executions: FixedTraceToolExecution[] = []; const completedExecutions: ToolExecution[] = []; const invocations: PreparedModelInvocation[] = []; + const providerExposures: FixedTraceProviderExposure[] = []; const seenCallIds = new Set(); const seenToolNames = new Set(); const modelLoop = new ModelTurnLoopState(iterationLimit); @@ -302,6 +314,7 @@ export async function executeFixedTraceToolLoop( usage: modelLoop.usage, tools: Object.freeze([...executions]), rejectedToolCalls: rejectedToolCalls(reason, calls), + providerExposures: Object.freeze([...providerExposures]), }) ); @@ -328,6 +341,15 @@ export async function executeFixedTraceToolLoop( await options.beforeDispatch?.(prepared); }, }); + const prepared = invocations.at(-1); + if (!prepared) throw new Error('fixed-trace model response was not preceded by a prepared invocation'); + providerExposures.push(Object.freeze({ + attempt: invocations.length, + preparedProvider: prepared.provider, + preparedModel: prepared.model, + returnedProvider: response.provider, + returnedModel: response.model, + })); const turn = activeTurn.acceptResponse(response); if (turn.providerToolCalls.length > 0 || turn.providerToolResults.length > 0) { @@ -350,6 +372,7 @@ export async function executeFixedTraceToolLoop( usage: modelLoop.usage, tools: Object.freeze([...executions]), invocations: Object.freeze([...invocations]), + providerExposures: Object.freeze([...providerExposures]), }; } diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index 36e195abc8..81492e8f4a 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -1,367 +1,32 @@ -/** - * Live synthetic fixed-trace replay across normalized providers. - * - * Production handlers and production messages are never loaded into the - * executor: every tool result comes from the immutable fixed-trace fixtures. - * The required shared soft budget admits each exact prepared request before - * dispatch and halts after unknown spend exposure. - * - * `--architecture-arm=direct_generation` is intentionally admission-only. - * Production builds an authorization-aware definition/handler intersection - * before intent narrowing, but this harness neither captures that intersection - * nor bounds it independently; fixture-local schemas must not stand in for it. - * `oracle_route_diagnostic` may execute generation with fixture routing. - * The hybrid-only `--suite=hybrid-evaluator` binds the separately reviewed - * local-admission corpus without altering the legacy 32 traces. Every arm is - * diagnostic-only in this foundation: independent judging, - * comparison, and rollout are blocked until an evaluator-owned run-context - * and raw-ledger coordinator can authenticate serialized artifacts. - * - * Example: - * DOTENV_CONFIG_PATH=.env.local npm run eval:addie-fixed-traces -- \ - * --soft-max-usd=1 --output=.context/evals/fixed-traces.json - */ -import { createHash, randomUUID } from 'node:crypto'; -import { execFileSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { ModelConfig } from '../../src/config/models.js'; -import { CODE_VERSION, computeRouterRulesHash } from '../../src/addie/config-version.js'; -import { - BudgetedFixedTraceProvider, - FixedTraceBudget, - fixedTraceResponsePricingPolicy, -} from '../../src/addie/eval/fixed-trace-budget.js'; -import { - type FixedTraceProviderStageConfig, -} from '../../src/addie/eval/fixed-trace-runner.js'; -import { - runFixedTraceDiagnosticArtifact, - type FixedTraceDiagnosticProviderPlan, -} from '../../src/addie/eval/fixed-trace-diagnostic-run.js'; -import { MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS } from '../../src/addie/eval/fixed-trace-tool-loop.js'; -import { parseFixedTraceDiagnosticCliArguments } from '../../src/addie/eval/fixed-trace-diagnostic-cli.js'; -import { reserveFixedTraceDiagnosticOutput } from '../../src/addie/eval/fixed-trace-diagnostic-output.js'; -import { fixedTraceCommonToolDefinitions } from '../../src/addie/eval/fixed-trace-architecture.js'; -import { canonicalFixedTraceToolDefinitions } from '../../src/addie/eval/fixed-trace-tools.js'; -import { - fixedTraceHybridPolicy, - type FixedTraceArchitectureArmId, -} from '../../src/addie/eval/fixed-trace-architecture.js'; -import { - FIXED_TRACE_SUITE, - FIXED_TRACE_HYBRID_EVALUATOR_SUITE, - fixedTraceSuiteSha256, - type FixedTracePricing, -} from '../../src/addie/eval/fixed-trace-suite.js'; -import { AnthropicRouterProvider } from '../../src/addie/model-providers/anthropic-router-provider.js'; -import { AnthropicModelProvider } from '../../src/addie/model-providers/anthropic-provider.js'; -import type { - ModelProvider, - ModelProviderId, - ModelReasoningEffort, -} from '../../src/addie/model-providers/model-provider.js'; -import { - OpenAIResponsesProvider, - OPENAI_ROUTER_MODEL, -} from '../../src/addie/model-providers/openai-responses-provider.js'; -import { - GoogleGenerateContentProvider, - GOOGLE_ROUTER_MODEL, -} from '../../src/addie/model-providers/google-generate-content-provider.js'; -import { loadResponseStyle, loadRules } from '../../src/addie/rules/index.js'; +/** Planning-only manual entrypoint: it has no dispatch or output path. */ +import { parseFixedTraceDiagnosticCliArguments } from "../../src/addie/eval/fixed-trace-diagnostic-cli.js"; -type ProviderName = ModelProviderId; - -type ProviderPlan = FixedTraceDiagnosticProviderPlan & { name: ProviderName }; - -const PRICING = { - anthropicRouter: { - profileId: 'anthropic-standard-2026-09:claude-haiku-4-5', - inputUsdPerMillionTokens: 1, - outputUsdPerMillionTokens: 5, - cacheReadUsdPerMillionTokens: 0.1, - cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Anthropic pricing page: Claude Haiku 4.5, checked 2026-09-05.', - }, - anthropicGeneration: { - profileId: 'anthropic-standard-2026-09:claude-sonnet-5', - inputUsdPerMillionTokens: 2, - outputUsdPerMillionTokens: 10, - cacheReadUsdPerMillionTokens: 0.2, - cacheWriteUsdPerMillionTokens: 2.5, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Anthropic pricing page: Claude Sonnet 5 standard (5-minute cache write), checked 2026-09-05.', - }, - openai: { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', - inputUsdPerMillionTokens: 0.2, - outputUsdPerMillionTokens: 1.2, - cacheReadUsdPerMillionTokens: 0.02, - cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'unsupported', - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', - }, - google: { - profileId: 'google-gemini-3.7-flash-through-2026-12-31', - inputUsdPerMillionTokens: 0.75, - outputUsdPerMillionTokens: 3.75, - cacheReadUsdPerMillionTokens: 0.075, - cacheWriteUsdPerMillionTokens: 0.75, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'additive', - source: 'Google Gemini 3.7 Flash introductory standard, checked 2026-08-25.', - }, -} satisfies Record; - -const cliArguments = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); - -function argument(name: string): string | undefined { - return cliArguments[{ providers: 'providers', 'architecture-arm': 'architectureArm', suite: 'suite', 'soft-max-usd': 'softMaxUsd', output: 'output' }[name] as keyof typeof cliArguments] as string | undefined; -} - -function sha256(value: string): string { - return createHash('sha256').update(value, 'utf8').digest('hex'); -} - -function sourceBundle(): { sha256: string; files: string[] } { - const trackedFiles = execFileSync('git', [ - 'ls-files', '-z', 'package.json', 'package-lock.json', 'server/src/addie', - 'server/src/config/models.ts', 'server/tests/manual/fixed-trace-provider-eval.ts', - ], { encoding: 'utf8' }).split('\0').filter(Boolean).sort(); - const files = [...new Set([ - ...trackedFiles, - 'server/src/addie/eval/fixed-trace-budget.ts', - 'server/src/addie/eval/fixed-trace-architecture.ts', - 'server/src/addie/eval/fixed-trace-runner.ts', - 'server/tests/manual/fixed-trace-provider-eval.ts', - ])].sort(); - const hash = createHash('sha256'); - for (const file of files) { - hash.update(file, 'utf8').update('\0').update(readFileSync(file)).update('\0'); - } - return { sha256: hash.digest('hex'), files }; -} - -function stage( - provider: ModelProvider, - model: string, - reasoningEffort: ModelReasoningEffort, - maxOutputTokens: number, - maxIterations: number, - pricing: FixedTracePricing, -): FixedTraceProviderStageConfig { - return { - provider, - model, - reasoningEffort, - maxOutputTokens, - timeoutMs: 120_000, - maxIterations, - transportRetries: 0, - samplingMode: 'provider_no_sampling_control', - temperature: null, - pricing, - }; -} - -function budgetedStageProvider( - provider: ModelProvider, - budget: FixedTraceBudget, - model: string, - pricing: FixedTracePricing, -): BudgetedFixedTraceProvider { - return new BudgetedFixedTraceProvider( - provider, - budget, - pricing, - fixedTraceResponsePricingPolicy(provider.id, model, pricing), +const arguments_ = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); +if (!arguments_.validateOnly) + throw new Error( + "This planning-only evaluator requires --validate-only and cannot dispatch providers", ); -} - -function providerPlans( - names: readonly ProviderName[], - budget: FixedTraceBudget, -): ProviderPlan[] { - const plans: ProviderPlan[] = []; - if (names.includes('anthropic')) { - if (!process.env.ANTHROPIC_API_KEY) throw new Error('ANTHROPIC_API_KEY is required'); - if (ModelConfig.fast !== 'claude-haiku-4-5') throw new Error('Fixed traces pin Anthropic routing to claude-haiku-4-5'); - if (ModelConfig.primary !== 'claude-sonnet-5') throw new Error('Fixed traces pin Anthropic generation to claude-sonnet-5'); - const router = new AnthropicRouterProvider(process.env.ANTHROPIC_API_KEY, { maxRetries: 0 }); - const generation = new AnthropicModelProvider( - process.env.ANTHROPIC_API_KEY, - undefined, - { transportMaxRetries: 0 }, - ); - const budgetedGeneration = budgetedStageProvider(generation, budget, ModelConfig.primary, PRICING.anthropicGeneration); - plans.push({ - name: 'anthropic', - router: stage( - budgetedStageProvider(router, budget, ModelConfig.fast, PRICING.anthropicRouter), - ModelConfig.fast, - 'provider_default', - 300, - 1, - PRICING.anthropicRouter, - ), - generation: stage( - budgetedGeneration, - ModelConfig.primary, - 'provider_default', - 900, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.anthropicGeneration, - ), - }); - } - if (names.includes('openai')) { - if (!process.env.OPENAI_API_KEY) throw new Error('OPENAI_API_KEY is required'); - const provider = new OpenAIResponsesProvider(process.env.OPENAI_API_KEY); - const budgetedProvider = budgetedStageProvider(provider, budget, OPENAI_ROUTER_MODEL, PRICING.openai); - plans.push({ - name: 'openai', - router: stage( - budgetedProvider, - OPENAI_ROUTER_MODEL, - 'none', - 300, - 1, - PRICING.openai, - ), - generation: stage( - budgetedProvider, - OPENAI_ROUTER_MODEL, - 'none', - 900, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.openai, - ), - }); - } - if (names.includes('google')) { - if (!process.env.GEMINI_API_KEY) throw new Error('GEMINI_API_KEY is required'); - const provider = new GoogleGenerateContentProvider(process.env.GEMINI_API_KEY); - const budgetedProvider = budgetedStageProvider(provider, budget, GOOGLE_ROUTER_MODEL, PRICING.google); - plans.push({ - name: 'google', - router: stage( - budgetedProvider, - GOOGLE_ROUTER_MODEL, - 'low', - 1_200, - 1, - PRICING.google, - ), - generation: stage( - budgetedProvider, - GOOGLE_ROUTER_MODEL, - 'low', - 1_200, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.google, - ), - }); - } - return plans; -} - -const providerNames = (argument('providers') ?? 'anthropic,openai,google').split(',') as ProviderName[]; -if (providerNames.some((name) => !['anthropic', 'openai', 'google'].includes(name))) { - throw new Error('Unknown --providers value'); -} -if (new Set(providerNames).size !== providerNames.length || providerNames.length === 0) { - throw new Error('--providers must contain one or more unique providers'); -} -const architectureArm = (argument('architecture-arm') ?? 'two_stage_llm_router') as FixedTraceArchitectureArmId; -if (!(architectureArm in { two_stage_llm_router: true, direct_generation: true, deterministic_policy_llm_fallback_hybrid: true, oracle_route_diagnostic: true })) { - throw new Error('Unknown --architecture-arm value'); -} -const suiteName = argument('suite') ?? 'canonical'; -if (suiteName !== 'canonical' && suiteName !== 'hybrid-evaluator') throw new Error('Unknown --suite value'); -if (suiteName === 'hybrid-evaluator' && architectureArm !== 'deterministic_policy_llm_fallback_hybrid') { - throw new Error('--suite=hybrid-evaluator requires --architecture-arm=deterministic_policy_llm_fallback_hybrid'); -} -const traceSuite = suiteName === 'hybrid-evaluator' - ? FIXED_TRACE_HYBRID_EVALUATOR_SUITE - : FIXED_TRACE_SUITE; -const softMaxUsd = Number(argument('soft-max-usd')); -if (!Number.isFinite(softMaxUsd) || softMaxUsd <= 0) { - throw new Error('--soft-max-usd is required and must be positive'); -} -const outputArgument = argument('output'); -if (!outputArgument?.trim()) throw new Error('--output is required'); -const outputPath = resolve(outputArgument); -if (cliArguments.validateOnly) { - console.log(JSON.stringify({ +// This entrypoint is deliberately data-only. Some transitive corpus modules +// initialize diagnostic loggers while their immutable declarations load; keep +// those process-local diagnostics isolated from the one-machine-readable-line +// validate-only contract. +process.env.LOG_LEVEL = "silent"; +const { + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + assertFixedTraceEvaluationProtocol, + estimateFixedTraceEvaluationProtocol, +} = await import("../../src/addie/eval/fixed-trace-evaluation-protocol.js"); +assertFixedTraceEvaluationProtocol(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); +const estimate = estimateFixedTraceEvaluationProtocol( + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, +); +console.log( + JSON.stringify({ diagnosticOnly: true, - judgeDispatch: 'blocked_pending_trusted_evaluator_owned_coordinator', - validated: { - providers: providerNames, - architectureArm, - suite: suiteName, - softMaxUsd, - outputPath, - }, - })); - process.exit(0); -} -// This exclusive create happens before source inspection, credentials, -// provider construction, or dispatch. Never unlink it: an empty file is the -// truthful crash/incomplete marker if later setup fails. -const outputReservation = reserveFixedTraceDiagnosticOutput(outputPath); - -const gitCommit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); -const gitDirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim().length > 0; -const sources = sourceBundle(); -const promptConfigVersion = sha256(JSON.stringify({ - codeVersion: CODE_VERSION, - routerRulesHash: computeRouterRulesHash(), - rules: loadRules(), - responseStyle: loadResponseStyle(), -})); -const toolDefinitions = architectureArm === 'oracle_route_diagnostic' - ? canonicalFixedTraceToolDefinitions(traceSuite) - : fixedTraceCommonToolDefinitions(architectureArm); -const budget = new FixedTraceBudget(softMaxUsd); -const plans = providerPlans(providerNames, budget); -const runStartedAt = new Date().toISOString(); -const runRootId = `fixed-trace-${runStartedAt}-${randomUUID()}`; -const artifact = await runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: sources.sha256, - gitCommit, - gitDirty, - promptConfigVersion, - traceSuite, - traceSuiteSha256: fixedTraceSuiteSha256(traceSuite), - toolDefinitions, - toolDefinitionProvenance: architectureArm === 'oracle_route_diagnostic' - ? 'fixture_local' - : 'evaluator_owned_common_tool_universe', - architectureArm, - ...(architectureArm === 'deterministic_policy_llm_fallback_hybrid' - ? { hybridPolicy: fixedTraceHybridPolicy() } - : {}), - }, - budget, - outputReservation, - runRootId, - runStartedAt, - sourceBundleFiles: sources.files, - budgetNote: 'Soft admission target: exact prepared-request bytes and the full output allowance are reserved before each dispatch. Remote work may continue after a client timeout; any dispatched call without terminal usage marks exposure unknown and blocks every later dispatch.', -}); -console.log(JSON.stringify({ - outputPath, - runRootId, - providers: providerNames, - suite: suiteName, - comparisonEligible: artifact.comparisonEligible, - rolloutPass: artifact.rolloutPass, - budget: artifact.budget, -}, null, 2)); + dispatchable: false, + outputWritten: false, + providerCalls: 0, + externalFinalN: estimate.externalFinalN, + totalCeilingUsd: estimate.totalCeilingUsd, + }), +); diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts index b98706c109..1ac49da8a9 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts @@ -1,59 +1,81 @@ -import { describe, expect, it } from 'vitest'; -import { execFileSync } from 'node:child_process'; -import { existsSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { parseFixedTraceDiagnosticCliArguments } from '../../../src/addie/eval/fixed-trace-diagnostic-cli.js'; +import { describe, expect, it } from "vitest"; +import { spawnSync } from "node:child_process"; +import { realpathSync } from "node:fs"; +import { resolve } from "node:path"; +import { parseFixedTraceDiagnosticCliArguments } from "../../../src/addie/eval/fixed-trace-diagnostic-cli.js"; -describe('fixed-trace diagnostic CLI parser', () => { - it('accepts only bounded dry-run forms', () => { - expect(parseFixedTraceDiagnosticCliArguments(['--validate-only', '--providers=openai'])) - .toEqual({ validateOnly: true, providers: 'openai', architectureArm: undefined, suite: undefined, softMaxUsd: undefined, output: undefined }); - expect(parseFixedTraceDiagnosticCliArguments(['--validate-only=true']).validateOnly).toBe(true); +const REPOSITORY_ROOT = process.cwd(); +const LOCAL_TSX_CLI = realpathSync(resolve(REPOSITORY_ROOT, "node_modules/.bin/tsx")); +const LOCAL_TSX_EXPECTED = resolve(REPOSITORY_ROOT, "node_modules/tsx/dist/cli.mjs"); +const PROVIDER_FREE_ENV = { PATH: process.env.PATH ?? "", NODE_ENV: "test" }; + +function runPlanningCli(arguments_: readonly string[]) { + // Execute Node against this checkout's resolved tsx CLI, never npx. The + // minimal environment deliberately omits every provider credential. + expect(LOCAL_TSX_CLI).toBe(LOCAL_TSX_EXPECTED); + return spawnSync( + process.execPath, + [LOCAL_TSX_CLI, "server/tests/manual/fixed-trace-provider-eval.ts", ...arguments_], + { cwd: REPOSITORY_ROOT, encoding: "utf8", env: PROVIDER_FREE_ENV }, + ); +} + +describe("fixed-trace diagnostic CLI parser", () => { + it("accepts only the bare pinned validation flag", () => { + expect(parseFixedTraceDiagnosticCliArguments([])).toEqual({ validateOnly: false }); + expect(parseFixedTraceDiagnosticCliArguments(["--validate-only"])) + .toEqual({ validateOnly: true }); }); it.each([ - ['--validate-only=false'], ['--validate-onl'], ['--providers=openai', '--providers=google'], - ['positional'], ['--judge-providers=openai'], ['--providers'], ['--suite=unknown'], - ])('rejects unsafe option input %j', (args) => { + ["--validate-only=false"], + ["--validate-only=true"], + ["--validate-onl"], + ["--providers=openai"], + ["--architecture-arm=direct_generation"], + ["--suite=canonical"], + ["--trusted-manifest=forged.json"], + ["--experiment-plan=forged.json"], + ["--soft-max-usd=NaN"], + ["--providers=openai", "--providers=google"], + ["positional"], + ["--judge-providers=openai"], + ["--providers"], + ["--suite=unknown"], + ])("rejects unsafe option input %j", (args) => { expect(() => parseFixedTraceDiagnosticCliArguments(args)).toThrow(); }); - it('validates a complete bare dry run without credentials, writes, or provider setup', () => { - const output = resolve('/tmp/fixed-trace-diagnostic-cli-no-write.json'); - const result = execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=direct_generation', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, OPENAI_API_KEY: '' } }); - const validated = result.split('\n').map((line) => { - try { return JSON.parse(line) as Record; } catch { return null; } - }).find((line) => line?.diagnosticOnly === true); + it("emits one clean JSON line without credentials, writes, provider setup, or dispatch", () => { + const result = runPlanningCli(["--validate-only"]); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + const lines = result.stdout.trim().split("\n"); + expect(lines).toHaveLength(1); + const validated = JSON.parse(lines[0]!) as Record; expect(validated).toMatchObject({ diagnosticOnly: true, - validated: { providers: ['openai'], architectureArm: 'direct_generation', suite: 'canonical', softMaxUsd: 1, outputPath: output }, + dispatchable: false, + outputWritten: false, + providerCalls: 0, }); - expect(existsSync(output)).toBe(false); - }); - - it('binds the reviewed hybrid evaluator suite only to the hybrid arm during validate-only planning', () => { - const output = resolve('/tmp/fixed-trace-diagnostic-cli-hybrid-suite-no-write.json'); - const result = execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=deterministic_policy_llm_fallback_hybrid', '--suite=hybrid-evaluator', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, OPENAI_API_KEY: '' } }); - expect(result).toContain('"suite":"hybrid-evaluator"'); - expect(() => execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=two_stage_llm_router', '--suite=hybrid-evaluator', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); - expect(existsSync(output)).toBe(false); - }); + }, 20_000); it.each([ - ['--soft-max-usd=0', '--output=/tmp/out.json'], - ['--soft-max-usd=1'], - ])('rejects incomplete dry run configuration', (...args) => { - expect(() => execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', ...args, - ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); - }); + ["--output=/tmp/out.json"], + ["--providers=openai"], + ["--architecture-arm=direct_generation"], + ["--suite=canonical"], + ["--trusted-manifest=forged.json"], + ["--soft-max-usd=NaN"], + ["--validate-only=false"], + ])( + "rejects unbound runtime option %j without stdout or side effects", + (...args) => { + const result = runPlanningCli(["--validate-only", ...args]); + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("accepts only the bare --validate-only flag"); + }, + ); }); diff --git a/server/tests/unit/addie/fixed-trace-dormant-dispatch.test.ts b/server/tests/unit/addie/fixed-trace-dormant-dispatch.test.ts new file mode 100644 index 0000000000..cc3b6dffc7 --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-dormant-dispatch.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { realpathSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { parseFixedTraceDiagnosticCliArguments } from '../../../src/addie/eval/fixed-trace-diagnostic-cli.js'; + +const REPOSITORY_ROOT = process.cwd(); +const LOCAL_TSX_CLI = realpathSync(resolve(REPOSITORY_ROOT, 'node_modules/.bin/tsx')); +const LOCAL_TSX_EXPECTED = resolve(REPOSITORY_ROOT, 'node_modules/tsx/dist/cli.mjs'); +const PROVIDER_FREE_ENV = { PATH: process.env.PATH ?? '', NODE_ENV: 'test' }; + +describe('fixed-trace dormant dispatch boundary', () => { + it('refuses every runtime control except the bare validation flag', () => { + expect(parseFixedTraceDiagnosticCliArguments([])).toEqual({ validateOnly: false }); + for (const values of [ + ['--providers=openai'], ['--output=/tmp/fixed-trace.json'], + ['--validate-only=true'], ['--validate-only', '--suite=canonical'], + ]) { + expect(() => parseFixedTraceDiagnosticCliArguments(values)).toThrow( + 'accepts only the bare --validate-only flag', + ); + } + }); + + it('runs the manual entrypoint only as declaration validation', () => { + // Execute Node against this checkout's resolved tsx CLI, never npx. The + // minimal environment deliberately omits every provider credential. + expect(LOCAL_TSX_CLI).toBe(LOCAL_TSX_EXPECTED); + const result = spawnSync(process.execPath, [ + LOCAL_TSX_CLI, 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', + ], { + cwd: REPOSITORY_ROOT, encoding: 'utf8', env: PROVIDER_FREE_ENV, + }); + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ + diagnosticOnly: true, dispatchable: false, outputWritten: false, providerCalls: 0, + }); + }); +}); diff --git a/server/tests/unit/addie/fixed-trace-judge.test.ts b/server/tests/unit/addie/fixed-trace-judge.test.ts index 7604e02f35..bade63e9cd 100644 --- a/server/tests/unit/addie/fixed-trace-judge.test.ts +++ b/server/tests/unit/addie/fixed-trace-judge.test.ts @@ -135,7 +135,7 @@ replayProtection.replayStatus`.trim().split("\n")); const closedDomains = [ [FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS.schemaVersion, ["addie-fixed-trace-sealed-evidence-v1"]], [FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS.invocation.stage, ["router", "generation", "judge", "simulator"]], - [FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS.timingAndOutcome.terminalStatus, ["complete", "ignored", "reacted", "refusal", "truncated", "empty", "malformed", "provider_error", "timeout_after_dispatch", "not_dispatched_budget", "not_admitted_architecture"]], + [FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS.timingAndOutcome.terminalStatus, ["complete", "ignored", "reacted", "refusal", "truncated", "empty", "malformed", "provider_error", "timeout_after_dispatch", "unknown_exposure", "not_dispatched_budget", "not_admitted_architecture"]], [FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS.timingAndOutcome.finishReason, ["stop", "tool_calls", "length", "refusal", "continue"]], [FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS.denominatorAndSequence.completeness, ["complete", "incomplete", "unknown_exposure"]], [FIXED_TRACE_SEALED_EVIDENCE_REQUIREMENTS.denominatorAndSequence.tamperClass, ["none", "omission", "insertion", "duplication", "substitution", "reordering"]], diff --git a/server/tests/unit/addie/fixed-trace-rollout.test.ts b/server/tests/unit/addie/fixed-trace-rollout.test.ts index 8b38c36fd1..781c87328a 100644 --- a/server/tests/unit/addie/fixed-trace-rollout.test.ts +++ b/server/tests/unit/addie/fixed-trace-rollout.test.ts @@ -50,6 +50,7 @@ const summary: FixedTraceSummary = { malformed: 0, provider_error: 1, timeout_after_dispatch: 0, + unknown_exposure: 0, not_dispatched_budget: 0, not_admitted_architecture: 0, }, diff --git a/server/tests/unit/addie/fixed-trace-runner.test.ts b/server/tests/unit/addie/fixed-trace-runner.test.ts index 75b99e808a..dc4ddb4b29 100644 --- a/server/tests/unit/addie/fixed-trace-runner.test.ts +++ b/server/tests/unit/addie/fixed-trace-runner.test.ts @@ -695,6 +695,66 @@ describe('fixed trace artifact runner', () => { .toEqual(expect.arrayContaining(['router_model_resolution_policy_mismatch', 'router_cost_provenance_missing'])); }); + it('keeps an unapproved dispatched generation exposure unknown for grading and budget admission', async () => { + const selectedTrace = trace('knowledge-task-model'); + const router = new ScriptedProvider([routeResponse('respond', ['knowledge'])]); + const generationDelegate = new ScriptedProvider([{ + ...response([{ type: 'text', text: 'Synthetic protocol explanation.' }]), + model: 'other-anthropic-model', + }]); + const budget = new FixedTraceBudget(1); + const pricing = stage(generationDelegate, 3).pricing; + const generation = new BudgetedFixedTraceProvider( + generationDelegate, + budget, + pricing, + fixedTraceResponsePricingPolicy('anthropic', 'claude-haiku-4-5', pricing), + ); + + const observation = await runFixedTraceCase(selectedTrace, config(router, generation)); + const grade = gradeFixedTrace(selectedTrace, observation); + + expect(observation).toMatchObject({ + terminalStage: 'generation', + terminalStatus: 'unknown_exposure', + finishReason: 'stop', + flagged: true, + metadata: { + generation: { + source: 'provider', + dispatched: true, + usage: { inputTokens: 10, outputTokens: 5 }, + returnedProvider: 'anthropic', + returnedModel: 'other-anthropic-model', + modelResolution: 'provider_canonicalized', + estimatedCostUsd: null, + pricingSource: null, + pricingProfileId: null, + providerExposures: [{ + attempt: 1, + preparedProvider: 'anthropic', + preparedModel: 'claude-haiku-4-5', + returnedProvider: 'anthropic', + returnedModel: 'other-anthropic-model', + }], + }, + }, + }); + expect(observation.terminalStatus).not.toBe('complete'); + expect(observation.terminalStatus).not.toBe('not_dispatched_budget'); + expect(grade).toMatchObject({ terminalFailure: true, deterministicPass: false }); + expect(grade.failures).toContain('terminal_status_unexpected'); + expect(budget.snapshot()).toMatchObject({ + accountedSpendUsd: 0, + reservedUsd: 0, + remainingUsd: null, + dispatchedCalls: 1, + completedCalls: 0, + budgetRejectedCalls: 0, + exposureUnknown: true, + }); + }); + it('records complete router and multi-turn generation provenance', async () => { const router = new ScriptedProvider([routeResponse('respond', ['knowledge'])]); const generation = new ScriptedProvider([ diff --git a/server/tests/unit/addie/fixed-trace-tool-loop.test.ts b/server/tests/unit/addie/fixed-trace-tool-loop.test.ts index 91b562fd50..246a1dc6e9 100644 --- a/server/tests/unit/addie/fixed-trace-tool-loop.test.ts +++ b/server/tests/unit/addie/fixed-trace-tool-loop.test.ts @@ -326,6 +326,38 @@ describe('executeFixedTraceToolLoop', () => { expect(create).toHaveBeenCalledOnce(); }); + it('snapshots the returned identity when a tool-input boundary follows it', async () => { + const returned = anthropicResponse([{ + type: 'tool_use', id: 'tool_1', name: 'search_docs', input: {}, + }], 'tool_use', 'msg_1'); + returned.model = 'claude-fallback-for-boundary'; + const provider = new AnthropicModelProvider('unused', { + beta: { messages: { create: vi.fn().mockResolvedValue(returned) } }, + } as AnthropicMessagesTransport); + + try { + await executeFixedTraceToolLoop( + provider, + request('claude-test'), + trace('knowledge-task-model'), + [tool('search_docs', ['query']), tool('get_doc')], + ); + throw new Error('Expected a tool-input boundary'); + } catch (error) { + expect(error).toBeInstanceOf(FixedTraceToolLoopBoundaryError); + const checkpoint = (error as FixedTraceToolLoopBoundaryError).checkpoint!; + expect(checkpoint.providerExposures).toEqual([{ + attempt: 1, + preparedProvider: 'anthropic', + preparedModel: 'claude-test', + returnedProvider: 'anthropic', + returnedModel: 'claude-fallback-for-boundary', + }]); + expect(Object.isFrozen(checkpoint.providerExposures)).toBe(true); + expect(Object.isFrozen(checkpoint.providerExposures[0])).toBe(true); + } + }); + it('rejects duplicate mutation names within one provider response before execution', async () => { const duplicateMutationTrace = structuredClone(trace('knowledge-task-model')); duplicateMutationTrace.toolFixtures = [ @@ -374,6 +406,34 @@ describe('executeFixedTraceToolLoop', () => { expect(create).toHaveBeenCalledOnce(); }); + it('keeps ordered exposures through a later duplicate-tool boundary', async () => { + const create = vi.fn() + .mockResolvedValueOnce(anthropicResponse([{ + type: 'tool_use', id: 'tool_1', name: 'search_docs', input: { query: 'task model' }, + }], 'tool_use', 'msg_1')) + .mockResolvedValueOnce(anthropicResponse([{ + type: 'tool_use', id: 'tool_2', name: 'search_docs', input: { query: 'task model' }, + }], 'tool_use', 'msg_2')); + const provider = new AnthropicModelProvider('unused', { + beta: { messages: { create } }, + } as AnthropicMessagesTransport); + + await expect(executeFixedTraceToolLoop( + provider, + request('claude-test'), + trace('knowledge-task-model'), + [tool('search_docs', ['query']), tool('get_doc')], + )).rejects.toMatchObject({ + reason: 'duplicate_tool_call', + checkpoint: { + providerExposures: [ + { attempt: 1, returnedProvider: 'anthropic', returnedModel: 'claude-test' }, + { attempt: 2, returnedProvider: 'anthropic', returnedModel: 'claude-test' }, + ], + }, + }); + }); + it('requires an exact one-to-one fixture and schema registry', async () => { const provider = new AnthropicModelProvider('unused', { beta: { messages: { create: vi.fn() } },