diff --git a/server/src/addie/direct-tool-universe.ts b/server/src/addie/direct-tool-universe.ts index 20ae74866c..ab2c2e4a4d 100644 --- a/server/src/addie/direct-tool-universe.ts +++ b/server/src/addie/direct-tool-universe.ts @@ -166,6 +166,7 @@ function captureFixedTraceEvaluatorToolUniverse(): CapturedDirectToolUniverse { */ export const FIXED_TRACE_DIRECT_TOOL_UNIVERSE = captureFixedTraceEvaluatorToolUniverse(); -export const FIXED_TRACE_DIRECT_TOOL_HANDLERS = createSyntheticDirectToolReceiptHandlers( - FIXED_TRACE_DIRECT_TOOL_UNIVERSE, -); +/** Construct inert handlers only when a direct replay explicitly asks for them. */ +export function fixedTraceDirectToolHandlers(): Map { + return createSyntheticDirectToolReceiptHandlers(FIXED_TRACE_DIRECT_TOOL_UNIVERSE); +} diff --git a/server/src/addie/eval/fixed-trace-architecture.ts b/server/src/addie/eval/fixed-trace-architecture.ts index 58afced384..b87c0d8a3f 100644 --- a/server/src/addie/eval/fixed-trace-architecture.ts +++ b/server/src/addie/eval/fixed-trace-architecture.ts @@ -1,8 +1,8 @@ -import type { AddieTool } from '../types.js'; -import { FIXED_TRACE_DIRECT_TOOL_UNIVERSE } from '../direct-tool-universe.js'; -import { quickMatchRoutingContext, type ExecutionPlan } from '../router.js'; -import { createHash } from 'node:crypto'; -import type { FixedTraceCase } from './fixed-trace-suite.js'; +import type { AddieTool } from "../types.js"; +import { FIXED_TRACE_DIRECT_TOOL_UNIVERSE } from "../direct-tool-universe.js"; +import { quickMatchRoutingContext, type ExecutionPlan } from "../router.js"; +import { createHash } from "node:crypto"; +import type { FixedTraceCase } from "./fixed-trace-suite.js"; /** * Architecture is a cohort boundary, not a tunable label. In particular, an @@ -11,34 +11,37 @@ import type { FixedTraceCase } from './fixed-trace-suite.js'; */ export const FIXED_TRACE_ARCHITECTURE_ARMS = Object.freeze({ two_stage_llm_router: Object.freeze({ - id: 'two_stage_llm_router', - routeSource: 'llm_router', + id: "two_stage_llm_router", + routeSource: "llm_router", // Architectural capability is distinct from authenticated evaluation // evidence; this foundation is diagnostic-only. rolloutEligible: false, diagnosticOnly: true, }), direct_generation: Object.freeze({ - id: 'direct_generation', - routeSource: 'deployable_surface_policy', + id: "direct_generation", + routeSource: "deployable_surface_policy", + admission: "not_admitted_architecture", rolloutEligible: false, diagnosticOnly: true, }), deterministic_policy_llm_fallback_hybrid: Object.freeze({ - id: 'deterministic_policy_llm_fallback_hybrid', - routeSource: 'reviewed_safe_subset_of_production_quick_match_with_unchanged_llm_fallback', + id: "deterministic_policy_llm_fallback_hybrid", + routeSource: + "reviewed_safe_subset_of_production_quick_match_with_unchanged_llm_fallback", rolloutEligible: false, diagnosticOnly: true, }), oracle_route_diagnostic: Object.freeze({ - id: 'oracle_route_diagnostic', - routeSource: 'fixture_oracle', + id: "oracle_route_diagnostic", + routeSource: "fixture_oracle", rolloutEligible: false, diagnosticOnly: true, }), } as const); -export type FixedTraceArchitectureArmId = keyof typeof FIXED_TRACE_ARCHITECTURE_ARMS; +export type FixedTraceArchitectureArmId = + keyof typeof FIXED_TRACE_ARCHITECTURE_ARMS; export type FixedTraceArchitectureArmProvenance = (typeof FIXED_TRACE_ARCHITECTURE_ARMS)[FixedTraceArchitectureArmId]; @@ -47,15 +50,17 @@ export type FixedTraceArchitectureArmProvenance = * quick-match policy. It can only terminate no-tool surface outcomes; every * routed/tool-bearing decision retains the incumbent strict LLM router. */ -export const FIXED_TRACE_HYBRID_POLICY_VERSION = 'fixed-trace-hybrid-safe-subset-v2'; -export const FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION = 'exact-harmless-terminal-admission-v1'; +export const FIXED_TRACE_HYBRID_POLICY_VERSION = + "fixed-trace-hybrid-safe-subset-v2"; +export const FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION = + "exact-harmless-terminal-admission-v1"; export class FixedTraceHybridAdmissionSnapshotError extends Error { - readonly code = 'invalid_hybrid_admission_snapshot'; + readonly code = "invalid_hybrid_admission_snapshot"; constructor(message: string, options?: ErrorOptions) { super(message, options); - this.name = 'FixedTraceHybridAdmissionSnapshotError'; + this.name = "FixedTraceHybridAdmissionSnapshotError"; } } @@ -63,46 +68,48 @@ export interface FixedTraceHybridPolicy { version: string; /** The reviewed fail-closed admission gate, bound into cohort provenance. */ safetyGateVersion: typeof FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION; - safetyGateStatus: 'reviewed_safe_subset'; + safetyGateStatus: "reviewed_safe_subset"; /** This arm never claims to run the full production substring matcher. */ - localAdmissionSource: 'reviewed_safe_subset_of_production_quick_match'; - fallbackSource: 'unchanged_incumbent_two_stage_llm_router'; + localAdmissionSource: "reviewed_safe_subset_of_production_quick_match"; + fallbackSource: "unchanged_incumbent_two_stage_llm_router"; /** A subset of production quick-match terminal actions, never `respond`. */ - localTerminalActions: readonly ('ignore' | 'react')[]; + localTerminalActions: readonly ("ignore" | "react")[]; /** Admin state is never an admission signal for a local outcome. */ requireNonAdmin: true; /** Channel outcomes require a captured private-channel fact. */ requirePrivateChannelForChannelOutcome: true; /** All non-local outcomes use the incumbent strict router stage. */ - fallbackRouter: 'two_stage_llm_router'; + fallbackRouter: "two_stage_llm_router"; } export interface FixedTraceHybridDecision { - mode: 'local_terminal' | 'llm_router_fallback'; + mode: "local_terminal" | "llm_router_fallback"; reason: - | 'production_quick_match_terminal' - | 'no_production_quick_match' - | 'thread_context_requires_router' - | 'admin_requires_router' - | 'channel_privacy_not_captured' - | 'unsafe_or_ambiguous_message' - | 'quick_match_exception' - | 'tool_or_mutation_capability_requires_router' - | 'policy_disallows_terminal_action'; + | "production_quick_match_terminal" + | "no_production_quick_match" + | "thread_context_requires_router" + | "admin_requires_router" + | "channel_privacy_not_captured" + | "unsafe_or_ambiguous_message" + | "quick_match_exception" + | "tool_or_mutation_capability_requires_router" + | "policy_disallows_terminal_action"; plan: ExecutionPlan | null; } -const DEFAULT_FIXED_TRACE_HYBRID_POLICY: FixedTraceHybridPolicy = Object.freeze({ - version: FIXED_TRACE_HYBRID_POLICY_VERSION, - safetyGateVersion: FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION, - safetyGateStatus: 'reviewed_safe_subset', - localAdmissionSource: 'reviewed_safe_subset_of_production_quick_match', - fallbackSource: 'unchanged_incumbent_two_stage_llm_router', - localTerminalActions: Object.freeze(['ignore', 'react'] as const), - requireNonAdmin: true, - requirePrivateChannelForChannelOutcome: true, - fallbackRouter: 'two_stage_llm_router', -}); +const DEFAULT_FIXED_TRACE_HYBRID_POLICY: FixedTraceHybridPolicy = Object.freeze( + { + version: FIXED_TRACE_HYBRID_POLICY_VERSION, + safetyGateVersion: FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION, + safetyGateStatus: "reviewed_safe_subset", + localAdmissionSource: "reviewed_safe_subset_of_production_quick_match", + fallbackSource: "unchanged_incumbent_two_stage_llm_router", + localTerminalActions: Object.freeze(["ignore", "react"] as const), + requireNonAdmin: true, + requirePrivateChannelForChannelOutcome: true, + fallbackRouter: "two_stage_llm_router", + }, +); export function fixedTraceHybridPolicy( policy: FixedTraceHybridPolicy | undefined = undefined, @@ -110,51 +117,72 @@ export function fixedTraceHybridPolicy( return policy ?? DEFAULT_FIXED_TRACE_HYBRID_POLICY; } -export function validateFixedTraceHybridPolicy(policy: FixedTraceHybridPolicy): void { - if (!policy.version.trim()) throw new Error('Fixed trace hybrid policy version is required'); +export function validateFixedTraceHybridPolicy( + policy: FixedTraceHybridPolicy, +): void { + if (!policy.version.trim()) + throw new Error("Fixed trace hybrid policy version is required"); if ( - !Array.isArray(policy.localTerminalActions) - || policy.localTerminalActions.length === 0 - || policy.localTerminalActions.some((action) => action !== 'ignore' && action !== 'react') - || new Set(policy.localTerminalActions).size !== policy.localTerminalActions.length - || policy.safetyGateVersion !== FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION - || policy.safetyGateStatus !== 'reviewed_safe_subset' - || policy.localAdmissionSource !== 'reviewed_safe_subset_of_production_quick_match' - || policy.fallbackSource !== 'unchanged_incumbent_two_stage_llm_router' - || policy.requireNonAdmin !== true - || policy.requirePrivateChannelForChannelOutcome !== true - || policy.fallbackRouter !== 'two_stage_llm_router' - ) throw new Error('Fixed trace hybrid policy is invalid'); + !Array.isArray(policy.localTerminalActions) || + policy.localTerminalActions.length === 0 || + policy.localTerminalActions.some( + (action) => action !== "ignore" && action !== "react", + ) || + new Set(policy.localTerminalActions).size !== + policy.localTerminalActions.length || + policy.safetyGateVersion !== FIXED_TRACE_HYBRID_SAFETY_GATE_VERSION || + policy.safetyGateStatus !== "reviewed_safe_subset" || + policy.localAdmissionSource !== + "reviewed_safe_subset_of_production_quick_match" || + policy.fallbackSource !== "unchanged_incumbent_two_stage_llm_router" || + policy.requireNonAdmin !== true || + policy.requirePrivateChannelForChannelOutcome !== true || + policy.fallbackRouter !== "two_stage_llm_router" + ) + throw new Error("Fixed trace hybrid policy is invalid"); } type HybridAdmissionSnapshot = Readonly<{ message: string; - source: FixedTraceCase['request']['source']; + source: FixedTraceCase["request"]["source"]; isAdmin: boolean; isThread: boolean; - channelPrivacy?: 'private' | 'public'; + channelPrivacy?: "private" | "public"; }>; -type HybridQuickMatcher = (context: Readonly<{ - message: string; - source: 'dm' | 'channel'; - isThread: boolean; - isAAOAdmin: boolean; -}>) => ExecutionPlan | null; - -function ownDataProperty(source: unknown, name: string, owner = 'input'): unknown { +type HybridQuickMatcher = ( + context: Readonly<{ + message: string; + source: "dm" | "channel"; + isThread: boolean; + isAAOAdmin: boolean; + }>, +) => ExecutionPlan | null; + +function ownDataProperty( + source: unknown, + name: string, + owner = "input", +): unknown { try { - if (typeof source !== 'object' || source === null) { - throw new FixedTraceHybridAdmissionSnapshotError(`Hybrid admission ${owner} must be an object`); + if (typeof source !== "object" || source === null) { + throw new FixedTraceHybridAdmissionSnapshotError( + `Hybrid admission ${owner} must be an object`, + ); } const descriptor = Object.getOwnPropertyDescriptor(source, name); - if (!descriptor || !('value' in descriptor)) { - throw new FixedTraceHybridAdmissionSnapshotError(`Hybrid admission ${owner}.${name} must be an own data property`); + if (!descriptor || !("value" in descriptor)) { + throw new FixedTraceHybridAdmissionSnapshotError( + `Hybrid admission ${owner}.${name} must be an own data property`, + ); } return descriptor.value; } catch (error) { if (error instanceof FixedTraceHybridAdmissionSnapshotError) throw error; - throw new FixedTraceHybridAdmissionSnapshotError(`Hybrid admission ${owner}.${name} could not be snapshotted`, { cause: error }); + throw new FixedTraceHybridAdmissionSnapshotError( + `Hybrid admission ${owner}.${name} could not be snapshotted`, + { cause: error }, + ); } } @@ -164,76 +192,146 @@ function ownDataProperty(source: unknown, name: string, owner = 'input'): unknow * provider dispatch rather than participating in routing. */ function snapshotHybridAdmissionInput(input: unknown): HybridAdmissionSnapshot { - const message = ownDataProperty(input, 'message'); - const source = ownDataProperty(input, 'source'); - const isAdmin = ownDataProperty(input, 'isAdmin'); - const isThread = ownDataProperty(input, 'isThread'); + const message = ownDataProperty(input, "message"); + const source = ownDataProperty(input, "source"); + const isAdmin = ownDataProperty(input, "isAdmin"); + const isThread = ownDataProperty(input, "isThread"); let channelPrivacy: unknown; try { - const descriptor = typeof input === 'object' && input !== null - ? Object.getOwnPropertyDescriptor(input, 'channelPrivacy') - : undefined; - if (descriptor && !('value' in descriptor)) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission input.channelPrivacy must be an own data property'); + const descriptor = + typeof input === "object" && input !== null + ? Object.getOwnPropertyDescriptor(input, "channelPrivacy") + : undefined; + if (descriptor && !("value" in descriptor)) { + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission input.channelPrivacy must be an own data property", + ); } channelPrivacy = descriptor?.value; } catch (error) { if (error instanceof FixedTraceHybridAdmissionSnapshotError) throw error; - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission input.channelPrivacy could not be snapshotted', { cause: error }); + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission input.channelPrivacy could not be snapshotted", + { cause: error }, + ); } if ( - typeof message !== 'string' - || (source !== 'dm' && source !== 'channel') - || typeof isAdmin !== 'boolean' - || typeof isThread !== 'boolean' - || (channelPrivacy !== undefined && channelPrivacy !== 'private' && channelPrivacy !== 'public') - ) throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission input has invalid request facts'); - return Object.freeze({ message, source, isAdmin, isThread, ...(channelPrivacy === undefined ? {} : { channelPrivacy }) }); + typeof message !== "string" || + (source !== "dm" && source !== "channel") || + typeof isAdmin !== "boolean" || + typeof isThread !== "boolean" || + (channelPrivacy !== undefined && + channelPrivacy !== "private" && + channelPrivacy !== "public") + ) + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission input has invalid request facts", + ); + return Object.freeze({ + message, + source, + isAdmin, + isThread, + ...(channelPrivacy === undefined ? {} : { channelPrivacy }), + }); } function snapshotHybridPolicy(input: unknown): FixedTraceHybridPolicy { - const policy = ownDataProperty(input, 'policy'); - const localTerminalActions = ownDataProperty(policy, 'localTerminalActions', 'policy'); + const policy = ownDataProperty(input, "policy"); + const localTerminalActions = ownDataProperty( + policy, + "localTerminalActions", + "policy", + ); if (!Array.isArray(localTerminalActions)) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission policy.localTerminalActions must be an array'); + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission policy.localTerminalActions must be an array", + ); } - const actionLength = ownDataProperty(localTerminalActions, 'length', 'policy.localTerminalActions'); - if (typeof actionLength !== 'number' || !Number.isSafeInteger(actionLength) || actionLength < 0 || actionLength > 2) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission policy.localTerminalActions has invalid length'); + const actionLength = ownDataProperty( + localTerminalActions, + "length", + "policy.localTerminalActions", + ); + if ( + typeof actionLength !== "number" || + !Number.isSafeInteger(actionLength) || + actionLength < 0 || + actionLength > 2 + ) { + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission policy.localTerminalActions has invalid length", + ); } - const actions = Array.from({ length: actionLength }, (_, index) => ( - ownDataProperty(localTerminalActions, String(index), 'policy.localTerminalActions') - )); + const actions = Array.from({ length: actionLength }, (_, index) => + ownDataProperty( + localTerminalActions, + String(index), + "policy.localTerminalActions", + ), + ); const snapshot = Object.freeze({ - version: ownDataProperty(policy, 'version', 'policy'), - safetyGateVersion: ownDataProperty(policy, 'safetyGateVersion', 'policy'), - safetyGateStatus: ownDataProperty(policy, 'safetyGateStatus', 'policy'), - localAdmissionSource: ownDataProperty(policy, 'localAdmissionSource', 'policy'), - fallbackSource: ownDataProperty(policy, 'fallbackSource', 'policy'), + version: ownDataProperty(policy, "version", "policy"), + safetyGateVersion: ownDataProperty(policy, "safetyGateVersion", "policy"), + safetyGateStatus: ownDataProperty(policy, "safetyGateStatus", "policy"), + localAdmissionSource: ownDataProperty( + policy, + "localAdmissionSource", + "policy", + ), + fallbackSource: ownDataProperty(policy, "fallbackSource", "policy"), localTerminalActions: Object.freeze(actions), - requireNonAdmin: ownDataProperty(policy, 'requireNonAdmin', 'policy'), - requirePrivateChannelForChannelOutcome: ownDataProperty(policy, 'requirePrivateChannelForChannelOutcome', 'policy'), - fallbackRouter: ownDataProperty(policy, 'fallbackRouter', 'policy'), + requireNonAdmin: ownDataProperty(policy, "requireNonAdmin", "policy"), + requirePrivateChannelForChannelOutcome: ownDataProperty( + policy, + "requirePrivateChannelForChannelOutcome", + "policy", + ), + fallbackRouter: ownDataProperty(policy, "fallbackRouter", "policy"), }) as FixedTraceHybridPolicy; try { validateFixedTraceHybridPolicy(snapshot); } catch (error) { - throw new FixedTraceHybridAdmissionSnapshotError('Hybrid admission policy is invalid', { cause: error }); + throw new FixedTraceHybridAdmissionSnapshotError( + "Hybrid admission policy is invalid", + { cause: error }, + ); } return snapshot; } -type SafeTerminalForm = Readonly<{ action: 'ignore' | 'react'; emoji?: string }>; +type SafeTerminalForm = Readonly<{ + action: "ignore" | "react"; + emoji?: string; +}>; const SAFE_IGNORE_FORMS = new Set([ - 'ok', 'okay', 'k', 'got it', 'cool', 'nice', 'lol', 'haha', 'sounds good', - 'will do', 'on it', 'done', 'working on it', + "ok", + "okay", + "k", + "got it", + "cool", + "nice", + "lol", + "haha", + "sounds good", + "will do", + "on it", + "done", + "working on it", ]); const SAFE_REACT_FORMS = new Map([ - ['hi', 'wave'], ['hello', 'wave'], ['hey', 'wave'], ['good morning', 'wave'], - ['good afternoon', 'wave'], ['howdy', 'wave'], ['thanks', 'heart'], ['thank you', 'heart'], + ["hi", "wave"], + ["hello", "wave"], + ["hey", "wave"], + ["good morning", "wave"], + ["good afternoon", "wave"], + ["howdy", "wave"], + ["thanks", "heart"], + ["thank you", "heart"], ]); -const UNSAFE_OR_AMBIGUOUS_LANGUAGE = /\b(?:no|not|never|don't|do\s+not|delete|remove|ship|send|invoice|billing|payment|account|admin|tool|generate|create|update|change|cancel|refund|user)\b/i; +const UNSAFE_OR_AMBIGUOUS_LANGUAGE = + /\b(?:no|not|never|don't|do\s+not|delete|remove|ship|send|invoice|billing|payment|account|admin|tool|generate|create|update|change|cancel|refund|user)\b/i; const UNSAFE_DELIMITER_OR_QUOTE = /["'`;,:|/\\]/; const CONTROL_OR_LINE_SEPARATOR = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; @@ -242,21 +340,30 @@ const CONTROL_OR_LINE_SEPARATOR = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; * accepts only fully consumed exact harmless forms after bounded normalization; * production's wider substring matcher remains unchanged and is not evidence. */ -function safeTerminalForm(snapshot: HybridAdmissionSnapshot): SafeTerminalForm | null { - if (Buffer.byteLength(snapshot.message, 'utf8') > 128) return null; - if (CONTROL_OR_LINE_SEPARATOR.test(snapshot.message) || UNSAFE_DELIMITER_OR_QUOTE.test(snapshot.message)) return null; +function safeTerminalForm( + snapshot: HybridAdmissionSnapshot, +): SafeTerminalForm | null { + if (Buffer.byteLength(snapshot.message, "utf8") > 128) return null; + if ( + CONTROL_OR_LINE_SEPARATOR.test(snapshot.message) || + UNSAFE_DELIMITER_OR_QUOTE.test(snapshot.message) + ) + return null; const normalized = snapshot.message - .normalize('NFKC') + .normalize("NFKC") .toLowerCase() .trim() .replace(/[\u2018\u2019]/g, "'") .replace(/[\u201c\u201d]/g, '"'); if (!normalized || UNSAFE_OR_AMBIGUOUS_LANGUAGE.test(normalized)) return null; - if (SAFE_IGNORE_FORMS.has(normalized) || (normalized.endsWith('.') && SAFE_IGNORE_FORMS.has(normalized.slice(0, -1)))) { - return Object.freeze({ action: 'ignore' }); + if ( + SAFE_IGNORE_FORMS.has(normalized) || + (normalized.endsWith(".") && SAFE_IGNORE_FORMS.has(normalized.slice(0, -1))) + ) { + return Object.freeze({ action: "ignore" }); } const emoji = SAFE_REACT_FORMS.get(normalized); - return emoji ? Object.freeze({ action: 'react', emoji }) : null; + return emoji ? Object.freeze({ action: "react", emoji }) : null; } /** @@ -267,10 +374,10 @@ function safeTerminalForm(snapshot: HybridAdmissionSnapshot): SafeTerminalForm | */ export function decideFixedTraceHybridRoute(input: { message: string; - source: FixedTraceCase['request']['source']; + source: FixedTraceCase["request"]["source"]; isAdmin: boolean; isThread: boolean; - channelPrivacy?: 'private' | 'public'; + channelPrivacy?: "private" | "public"; policy: FixedTraceHybridPolicy; /** Internal test seam; production uses the unchanged quick-match function. */ quickMatcher?: HybridQuickMatcher; @@ -279,83 +386,143 @@ export function decideFixedTraceHybridRoute(input: { // Request facts are snapshotted before policy validation or matcher code. // A hostile request accessor therefore cannot run after a dispatch boundary. const policy = snapshotHybridPolicy(input); - if (snapshot.isAdmin) return { mode: 'llm_router_fallback', reason: 'admin_requires_router', plan: null }; - if (snapshot.isThread) return { mode: 'llm_router_fallback', reason: 'thread_context_requires_router', plan: null }; - if (snapshot.source === 'channel' && snapshot.channelPrivacy !== 'private') { - return { mode: 'llm_router_fallback', reason: 'channel_privacy_not_captured', plan: null }; + if (snapshot.isAdmin) + return { + mode: "llm_router_fallback", + reason: "admin_requires_router", + plan: null, + }; + if (snapshot.isThread) + return { + mode: "llm_router_fallback", + reason: "thread_context_requires_router", + plan: null, + }; + if (snapshot.source === "channel" && snapshot.channelPrivacy !== "private") { + return { + mode: "llm_router_fallback", + reason: "channel_privacy_not_captured", + plan: null, + }; } const safeForm = safeTerminalForm(snapshot); - if (!safeForm) return { mode: 'llm_router_fallback', reason: 'unsafe_or_ambiguous_message', plan: null }; + if (!safeForm) + return { + mode: "llm_router_fallback", + reason: "unsafe_or_ambiguous_message", + plan: null, + }; let matcher: HybridQuickMatcher; try { - const suppliedMatcher = Object.getOwnPropertyDescriptor(input, 'quickMatcher'); - if (suppliedMatcher && !('value' in suppliedMatcher)) throw new Error('quickMatcher accessor'); - if (suppliedMatcher?.value !== undefined && typeof suppliedMatcher.value !== 'function') { - throw new Error('quickMatcher is not a function'); + const suppliedMatcher = Object.getOwnPropertyDescriptor( + input, + "quickMatcher", + ); + if (suppliedMatcher && !("value" in suppliedMatcher)) + throw new Error("quickMatcher accessor"); + if ( + suppliedMatcher?.value !== undefined && + typeof suppliedMatcher.value !== "function" + ) { + throw new Error("quickMatcher is not a function"); } matcher = suppliedMatcher?.value ?? quickMatchRoutingContext; } catch { - return { mode: 'llm_router_fallback', reason: 'quick_match_exception', plan: null }; + return { + mode: "llm_router_fallback", + reason: "quick_match_exception", + plan: null, + }; } - let matchedAction: 'ignore' | 'react' | 'respond' | null; + let matchedAction: "ignore" | "react" | "respond" | null; let matchedEmoji: string | undefined; try { - const plan = matcher(Object.freeze({ - message: snapshot.message, - source: snapshot.source, - isThread: snapshot.isThread, - isAAOAdmin: snapshot.isAdmin, - })); + const plan = matcher( + Object.freeze({ + message: snapshot.message, + source: snapshot.source, + isThread: snapshot.isThread, + isAAOAdmin: snapshot.isAdmin, + }), + ); matchedAction = plan?.action ?? null; - matchedEmoji = plan?.action === 'react' ? plan.emoji : undefined; + matchedEmoji = plan?.action === "react" ? plan.emoji : undefined; } catch { - return { mode: 'llm_router_fallback', reason: 'quick_match_exception', plan: null }; + return { + mode: "llm_router_fallback", + reason: "quick_match_exception", + plan: null, + }; } - if (!matchedAction) return { mode: 'llm_router_fallback', reason: 'no_production_quick_match', plan: null }; - if (matchedAction === 'respond') { - return { mode: 'llm_router_fallback', reason: 'tool_or_mutation_capability_requires_router', plan: null }; + if (!matchedAction) + return { + mode: "llm_router_fallback", + reason: "no_production_quick_match", + plan: null, + }; + if (matchedAction === "respond") { + return { + mode: "llm_router_fallback", + reason: "tool_or_mutation_capability_requires_router", + plan: null, + }; } - if (matchedAction !== safeForm.action || !policy.localTerminalActions.includes(matchedAction)) { - return { mode: 'llm_router_fallback', reason: 'policy_disallows_terminal_action', plan: null }; + if ( + matchedAction !== safeForm.action || + !policy.localTerminalActions.includes(matchedAction) + ) { + return { + mode: "llm_router_fallback", + reason: "policy_disallows_terminal_action", + plan: null, + }; } - if (matchedAction === 'react' && matchedEmoji !== safeForm.emoji) { - return { mode: 'llm_router_fallback', reason: 'policy_disallows_terminal_action', plan: null }; + if (matchedAction === "react" && matchedEmoji !== safeForm.emoji) { + return { + mode: "llm_router_fallback", + reason: "policy_disallows_terminal_action", + plan: null, + }; } return { - mode: 'local_terminal', - reason: 'production_quick_match_terminal', - plan: safeForm.action === 'react' - ? Object.freeze({ - action: 'react' as const, - emoji: safeForm.emoji!, - reason: 'Reviewed exact harmless terminal form', - decision_method: 'quick_match' as const, - }) - : Object.freeze({ - action: 'ignore' as const, - reason: 'Reviewed exact harmless terminal form', - decision_method: 'quick_match' as const, - }), + mode: "local_terminal", + reason: "production_quick_match_terminal", + plan: + safeForm.action === "react" + ? Object.freeze({ + action: "react" as const, + emoji: safeForm.emoji!, + reason: "Reviewed exact harmless terminal form", + decision_method: "quick_match" as const, + }) + : Object.freeze({ + action: "ignore" as const, + reason: "Reviewed exact harmless terminal form", + decision_method: "quick_match" as const, + }), }; } export function fixedTraceArchitectureArm( - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceArchitectureArmProvenance { return FIXED_TRACE_ARCHITECTURE_ARMS[arm]; } export type FixedTraceToolDefinitionProvenance = - | 'fixture_local' - | 'evaluator_owned_production_definitions_simulated_receipts'; + "fixture_local" | "evaluator_owned_production_definitions_simulated_receipts"; /** Records what selected the candidate's visible tools for diagnostic replay. */ export interface FixedTraceToolUniverseProvenance { source: - | 'fixture_local_routed_replay' - | 'evaluator_owned_production_definitions_simulated_receipts' - | 'fixture_oracle'; - intentNarrowing: 'llm_router' | 'production_quick_match_or_llm_router' | 'not_applied' | 'fixture_oracle'; + | "fixture_local_routed_replay" + | "evaluator_owned_production_definitions_simulated_receipts" + | "fixture_oracle"; + intentNarrowing: + | "llm_router" + | "production_quick_match_or_llm_router" + | "not_applied" + | "fixture_oracle"; bounded: boolean; deployable: boolean; toolNames: readonly string[] | null; @@ -365,62 +532,79 @@ export interface FixedTraceToolUniverseProvenance { } export interface FixedTraceRequestThreadFactsProvenance { - source: 'not_applicable' | 'fixture_case_request_not_authenticated'; + source: "not_applicable" | "fixture_case_request_not_authenticated"; traceFacts: readonly Readonly<{ traceId: string; requestThreadFactsSha256: string; - provenance: 'fixture_case_request_not_authenticated'; + provenance: "fixture_case_request_not_authenticated"; }>[]; } export interface FixedTraceDirectRequestThreadFacts { - source: FixedTraceCase['request']['source']; + source: FixedTraceCase["request"]["source"]; isAAOAdmin: boolean; isThread: boolean; - channelPrivacy: 'private' | 'unknown'; - authentication: 'not_authenticated_fixture_claim'; - provenance: 'fixture_case_request_not_authenticated'; + channelPrivacy: "private" | "unknown"; + authentication: "not_authenticated_fixture_claim"; + provenance: "fixture_case_request_not_authenticated"; } function canonicalJson(value: unknown): string { - if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); - if (typeof value === 'number') return JSON.stringify(value); - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; - if (typeof value === 'object') { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (typeof value === "number") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { const record = value as Record; - return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; } - throw new Error('Cannot canonicalize a non-JSON request/thread fact'); + throw new Error("Cannot canonicalize a non-JSON request/thread fact"); } function sha256(value: unknown): string { - return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex'); + return createHash("sha256") + .update(canonicalJson(value), "utf8") + .digest("hex"); } /** Preserve fixture-visible facts exactly; never manufacture production auth/context. */ -export function fixedTraceDirectRequestThreadFacts(trace: FixedTraceCase): FixedTraceDirectRequestThreadFacts { +export function fixedTraceDirectRequestThreadFacts( + trace: FixedTraceCase, +): FixedTraceDirectRequestThreadFacts { return Object.freeze({ source: trace.request.source, isAAOAdmin: trace.request.isAdmin, isThread: (trace.request.threadContext?.length ?? 0) > 0, - channelPrivacy: trace.request.source === 'dm' ? 'private' : 'unknown', - authentication: 'not_authenticated_fixture_claim', - provenance: 'fixture_case_request_not_authenticated', + channelPrivacy: trace.request.source === "dm" ? "private" : "unknown", + authentication: "not_authenticated_fixture_claim", + provenance: "fixture_case_request_not_authenticated", }); } export function fixedTraceRequestThreadFactsProvenance( traceSuite: ReadonlyArray, - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceRequestThreadFactsProvenance { - if (arm !== 'direct_generation') return Object.freeze({ source: 'not_applicable', traceFacts: [] }); + if (arm !== "direct_generation") + return Object.freeze({ source: "not_applicable", traceFacts: [] }); return Object.freeze({ - source: 'fixture_case_request_not_authenticated', - traceFacts: Object.freeze(traceSuite.map((trace) => Object.freeze({ - traceId: trace.id, - requestThreadFactsSha256: sha256(fixedTraceDirectRequestThreadFacts(trace)), - provenance: 'fixture_case_request_not_authenticated' as const, - })).sort((left, right) => left.traceId.localeCompare(right.traceId))), + source: "fixture_case_request_not_authenticated", + traceFacts: Object.freeze( + traceSuite + .map((trace) => + Object.freeze({ + traceId: trace.id, + requestThreadFactsSha256: sha256( + fixedTraceDirectRequestThreadFacts(trace), + ), + provenance: "fixture_case_request_not_authenticated" as const, + }), + ) + .sort((left, right) => left.traceId.localeCompare(right.traceId)), + ), }); } @@ -430,60 +614,67 @@ export function fixedTraceRequestThreadFactsProvenance( * can reuse the production-equivalent executor. */ export interface FixedTraceExecutionEnvelopeProvenance { - source: 'fixture_expectation' | 'request_thread_facts_not_captured' | 'evaluator_owned_shared_request_thread_envelope' | 'fixture_oracle'; + source: + | "fixture_expectation" + | "request_thread_facts_not_captured" + | "evaluator_owned_shared_request_thread_envelope" + | "fixture_oracle"; deployable: boolean; } export function fixedTraceExecutionEnvelopeProvenance( - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceExecutionEnvelopeProvenance { - if (arm === 'direct_generation') return Object.freeze({ - source: 'evaluator_owned_shared_request_thread_envelope', - deployable: false, - }); - if (arm === 'oracle_route_diagnostic') return Object.freeze({ - source: 'fixture_oracle', - deployable: false, - }); - return Object.freeze({ source: 'fixture_expectation', deployable: false }); + if (arm === "direct_generation") + return Object.freeze({ + source: "evaluator_owned_shared_request_thread_envelope", + deployable: false, + }); + if (arm === "oracle_route_diagnostic") + return Object.freeze({ + source: "fixture_oracle", + deployable: false, + }); + return Object.freeze({ source: "fixture_expectation", deployable: false }); } export function fixedTraceToolUniverseProvenance( - arm: FixedTraceArchitectureArmId = 'two_stage_llm_router', + arm: FixedTraceArchitectureArmId = "two_stage_llm_router", ): FixedTraceToolUniverseProvenance { - if (arm === 'direct_generation') { + if (arm === "direct_generation") { return Object.freeze({ - source: 'evaluator_owned_production_definitions_simulated_receipts', - intentNarrowing: 'not_applied', + source: "evaluator_owned_production_definitions_simulated_receipts", + intentNarrowing: "not_applied", bounded: true, deployable: false, toolNames: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolNames, toolNamesSha256: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolNamesSha256, toolSchemaSha256: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolSchemaSha256, - definitionHandlerSha256: FIXED_TRACE_DIRECT_TOOL_UNIVERSE.definitionHandlerSha256, + definitionHandlerSha256: + FIXED_TRACE_DIRECT_TOOL_UNIVERSE.definitionHandlerSha256, }); } - if (arm === 'oracle_route_diagnostic') { + if (arm === "oracle_route_diagnostic") { return Object.freeze({ - source: 'fixture_oracle', - intentNarrowing: 'fixture_oracle', + source: "fixture_oracle", + intentNarrowing: "fixture_oracle", bounded: true, deployable: false, toolNames: null, }); } - if (arm === 'deterministic_policy_llm_fallback_hybrid') { + if (arm === "deterministic_policy_llm_fallback_hybrid") { return Object.freeze({ - source: 'fixture_local_routed_replay', - intentNarrowing: 'production_quick_match_or_llm_router', + source: "fixture_local_routed_replay", + intentNarrowing: "production_quick_match_or_llm_router", bounded: true, deployable: false, toolNames: null, }); } return Object.freeze({ - source: 'fixture_local_routed_replay', - intentNarrowing: 'llm_router', + source: "fixture_local_routed_replay", + intentNarrowing: "llm_router", bounded: true, deployable: false, toolNames: null, @@ -491,19 +682,19 @@ export function fixedTraceToolUniverseProvenance( } export type FixedTraceDirectArmAdmissionReason = - | 'fixture_local_tool_definitions' - | 'request_thread_execution_envelope_not_captured' - | 'production_binding_contract_not_captured' - | 'request_thread_facts_not_authenticated' - | 'evaluator_simulated_receipt_handlers'; + | "fixture_local_tool_definitions" + | "request_thread_execution_envelope_not_captured" + | "production_binding_contract_not_captured" + | "request_thread_facts_not_authenticated" + | "evaluator_simulated_receipt_handlers"; export interface FixedTraceDirectToolUniverse extends FixedTraceToolUniverseProvenance { - surface: FixedTraceCase['request']['source']; + surface: FixedTraceCase["request"]["source"]; isAdmin: boolean; isThread: boolean; - channelPrivacy: 'private' | 'unknown'; + channelPrivacy: "private" | "unknown"; requestThreadFactsSha256: string; - requestThreadFactsProvenance: 'fixture_case_request_not_authenticated'; + requestThreadFactsProvenance: "fixture_case_request_not_authenticated"; } export interface FixedTraceDirectArmAdmission { @@ -522,7 +713,10 @@ function freezeAdmission( reasons: Object.freeze([...reasons]), universe: Object.freeze({ ...universe, - toolNames: universe.toolNames === null ? null : Object.freeze([...universe.toolNames]), + toolNames: + universe.toolNames === null + ? null + : Object.freeze([...universe.toolNames]), }), }); } @@ -533,10 +727,12 @@ function freezeAdmission( * grades. The evaluator cannot yet capture the authenticated definition / * handler intersection, and must not substitute a fixture-local subset. */ -export function deriveFixedTraceDirectToolUniverse(trace: FixedTraceCase): FixedTraceDirectToolUniverse { +export function deriveFixedTraceDirectToolUniverse( + trace: FixedTraceCase, +): FixedTraceDirectToolUniverse { const facts = fixedTraceDirectRequestThreadFacts(trace); return Object.freeze({ - ...fixedTraceToolUniverseProvenance('direct_generation'), + ...fixedTraceToolUniverseProvenance("direct_generation"), // These remain fixture claims, not production authentication. They are // retained for audit and bound to the cohort, never replaced by a DM. surface: facts.source, @@ -567,9 +763,13 @@ export function admitFixedTraceDirectArm( // contract or request/thread envelope, so it must reject before dispatch. void definitions; void definitionProvenance; - return freezeAdmission(false, [ - 'production_binding_contract_not_captured', - 'request_thread_facts_not_authenticated', - 'evaluator_simulated_receipt_handlers', - ], universe); + return freezeAdmission( + false, + [ + "production_binding_contract_not_captured", + "request_thread_facts_not_authenticated", + "evaluator_simulated_receipt_handlers", + ], + universe, + ); } diff --git a/server/src/addie/eval/fixed-trace-budget.ts b/server/src/addie/eval/fixed-trace-budget.ts index 52a7a12df6..e01102b465 100644 --- a/server/src/addie/eval/fixed-trace-budget.ts +++ b/server/src/addie/eval/fixed-trace-budget.ts @@ -6,13 +6,16 @@ import type { ModelUsage, NormalizedModelEvent, PreparedModelInvocation, -} from '../model-providers/model-provider.js'; +} from "../model-providers/model-provider.js"; import { GOOGLE_ROUTER_MODEL, isGoogleRouterModelRevision, -} from '../model-providers/google-generate-content-provider.js'; -import { GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION } from '../model-cost-pricing.js'; -import type { FixedTraceModelResolutionPolicy } from './fixed-trace-suite.js'; +} from "../model-providers/google-generate-content-provider.js"; +import { + GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_LUNA_PRICING, +} from "../model-cost-pricing.js"; +import type { FixedTraceModelResolutionPolicy } from "./fixed-trace-suite.js"; export interface FixedTraceBudgetPricing { inputUsdPerMillionTokens: number; @@ -22,29 +25,28 @@ export interface FixedTraceBudgetPricing { /** Null means this provider does not expose a separately billable cache write rate. */ cacheWriteUsdPerMillionTokens?: number | null; /** Cache reads and writes have independently recorded provider semantics. */ - cacheReadAccounting?: 'additive' | 'subset' | 'unsupported'; - cacheWriteAccounting?: 'additive' | 'subset' | 'unsupported'; + cacheReadAccounting?: "additive" | "subset" | "unsupported"; + cacheWriteAccounting?: "additive" | "subset" | "unsupported"; source: string; } export type FixedTraceBudgetRejectionReason = - | 'budget_exposure_unknown' - | 'soft_limit_exceeded'; + "budget_exposure_unknown" | "soft_limit_exceeded"; export class FixedTraceBudgetAdmissionError extends Error { - readonly terminalStatus = 'not_dispatched_budget' as const; + readonly terminalStatus = "not_dispatched_budget" as const; constructor( readonly reason: FixedTraceBudgetRejectionReason, readonly prepared: PreparedModelInvocation, ) { super(reason); - this.name = 'FixedTraceBudgetAdmissionError'; + this.name = "FixedTraceBudgetAdmissionError"; } } export interface FixedTraceBudgetSnapshot { - policy: 'soft_admission_target'; + policy: "soft_admission_target"; softMaxUsd: number; accountedSpendUsd: number; reservedUsd: number; @@ -63,58 +65,74 @@ export interface FixedTraceBudgetSnapshot { */ interface FixedTraceApprovedPricing extends FixedTraceBudgetPricing { readonly profileId: string; - readonly expectedProvider: ModelProvider['id']; + readonly expectedProvider: ModelProvider["id"]; readonly expectedModel: string; readonly modelResolutionPolicy: FixedTraceModelResolutionPolicy; } export function fixedTraceModelResolutionPolicy( - provider: ModelProvider['id'], + provider: ModelProvider["id"], model: string, ): FixedTraceModelResolutionPolicy { - return provider === 'google' && model === GOOGLE_ROUTER_MODEL - ? 'google_router_dated_revision_v1' - : 'exact_model_identity_v1'; + return provider === "google" && model === GOOGLE_ROUTER_MODEL + ? "google_router_dated_revision_v1" + : "exact_model_identity_v1"; } -const FIXED_TRACE_APPROVED_PRICING = Object.freeze(([ - { - expectedProvider: 'anthropic', expectedModel: 'claude-haiku-4-5', - profileId: 'anthropic-standard-2026-08:claude-haiku-4-5', - inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, - cacheReadUsdPerMillionTokens: 0.1, cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.', - modelResolutionPolicy: 'exact_model_identity_v1', - }, - { - expectedProvider: 'anthropic', expectedModel: 'claude-sonnet-5', - profileId: 'anthropic-standard-2026-08:claude-sonnet-5', - inputUsdPerMillionTokens: 3, outputUsdPerMillionTokens: 15, - cacheReadUsdPerMillionTokens: 0.3, cacheWriteUsdPerMillionTokens: 3.75, - cacheReadAccounting: 'additive', cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Sonnet 5 standard, refreshed August 2026.', - modelResolutionPolicy: 'exact_model_identity_v1', - }, - { - expectedProvider: 'openai', expectedModel: 'gpt-5.6-luna', - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', - inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, - cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset', cacheWriteAccounting: 'unsupported', - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', - modelResolutionPolicy: 'exact_model_identity_v1', - }, - { - expectedProvider: 'google', expectedModel: GOOGLE_ROUTER_MODEL, - profileId: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - inputUsdPerMillionTokens: 0.75, outputUsdPerMillionTokens: 3.75, - cacheReadUsdPerMillionTokens: 0.075, cacheWriteUsdPerMillionTokens: 0.75, - cacheReadAccounting: 'subset', cacheWriteAccounting: 'additive', - source: 'Google Gemini 3.7 Flash introductory standard, checked 2026-08-25.', - modelResolutionPolicy: 'google_router_dated_revision_v1', - }, -] satisfies readonly FixedTraceApprovedPricing[]).map((entry) => Object.freeze(entry))); +const FIXED_TRACE_APPROVED_PRICING = Object.freeze( + ( + [ + { + expectedProvider: "anthropic", + expectedModel: "claude-haiku-4-5", + profileId: "anthropic-standard-2026-08:claude-haiku-4-5", + inputUsdPerMillionTokens: 1, + outputUsdPerMillionTokens: 5, + cacheReadUsdPerMillionTokens: 0.1, + cacheWriteUsdPerMillionTokens: 1.25, + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + source: + "Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.", + modelResolutionPolicy: "exact_model_identity_v1", + }, + { + expectedProvider: "anthropic", + expectedModel: "claude-sonnet-5", + profileId: "anthropic-standard-2026-08:claude-sonnet-5", + inputUsdPerMillionTokens: 3, + outputUsdPerMillionTokens: 15, + cacheReadUsdPerMillionTokens: 0.3, + cacheWriteUsdPerMillionTokens: 3.75, + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + source: + "Repository Anthropic pricing table: Claude Sonnet 5 standard, refreshed August 2026.", + modelResolutionPolicy: "exact_model_identity_v1", + }, + { + expectedProvider: "openai", + expectedModel: "gpt-5.6-luna", + ...OPENAI_GPT_5_6_LUNA_PRICING, + modelResolutionPolicy: "exact_model_identity_v1", + }, + { + expectedProvider: "google", + expectedModel: GOOGLE_ROUTER_MODEL, + profileId: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + inputUsdPerMillionTokens: 0.75, + outputUsdPerMillionTokens: 3.75, + cacheReadUsdPerMillionTokens: 0.075, + cacheWriteUsdPerMillionTokens: 0.75, + cacheReadAccounting: "subset", + cacheWriteAccounting: "additive", + source: + "Google Gemini 3.7 Flash introductory standard, checked 2026-08-25.", + modelResolutionPolicy: "google_router_dated_revision_v1", + }, + ] satisfies readonly FixedTraceApprovedPricing[] + ).map((entry) => Object.freeze(entry)), +); /** * The complete live approval surface. It is intentionally inspectable for @@ -122,22 +140,24 @@ const FIXED_TRACE_APPROVED_PRICING = Object.freeze(([ * these descriptive values. */ export function fixedTraceApprovedPricingProfiles(): readonly Readonly<{ - expectedProvider: ModelProvider['id']; + expectedProvider: ModelProvider["id"]; expectedModel: string; profileId: string; source: string; }>[] { - return FIXED_TRACE_APPROVED_PRICING.map((entry) => Object.freeze({ - expectedProvider: entry.expectedProvider, - expectedModel: entry.expectedModel, - profileId: entry.profileId, - source: entry.source, - })); + return FIXED_TRACE_APPROVED_PRICING.map((entry) => + Object.freeze({ + expectedProvider: entry.expectedProvider, + expectedModel: entry.expectedModel, + profileId: entry.profileId, + source: entry.source, + }), + ); } /** Opaque, module-branded policy produced only from the approved registry. */ export interface FixedTraceResponsePricingPolicy { - readonly expectedProvider: ModelProvider['id']; + readonly expectedProvider: ModelProvider["id"]; readonly expectedModel: string; readonly pricingProfileId: string; readonly modelResolutionPolicy: FixedTraceModelResolutionPolicy; @@ -152,36 +172,46 @@ function sameApprovedPricing( entry: FixedTraceApprovedPricing, pricing: FixedTraceBudgetPricing & { readonly profileId: string }, ): boolean { - return entry.profileId === pricing.profileId - && entry.inputUsdPerMillionTokens === pricing.inputUsdPerMillionTokens - && entry.outputUsdPerMillionTokens === pricing.outputUsdPerMillionTokens - && entry.cacheReadUsdPerMillionTokens === pricing.cacheReadUsdPerMillionTokens - && entry.cacheWriteUsdPerMillionTokens === pricing.cacheWriteUsdPerMillionTokens - && entry.cacheReadAccounting === pricing.cacheReadAccounting - && entry.cacheWriteAccounting === pricing.cacheWriteAccounting - && entry.source === pricing.source; + return ( + entry.profileId === pricing.profileId && + entry.inputUsdPerMillionTokens === pricing.inputUsdPerMillionTokens && + entry.outputUsdPerMillionTokens === pricing.outputUsdPerMillionTokens && + entry.cacheReadUsdPerMillionTokens === + pricing.cacheReadUsdPerMillionTokens && + entry.cacheWriteUsdPerMillionTokens === + pricing.cacheWriteUsdPerMillionTokens && + entry.cacheReadAccounting === pricing.cacheReadAccounting && + entry.cacheWriteAccounting === pricing.cacheWriteAccounting && + entry.source === pricing.source + ); } function approvedResponsePricing( policy: FixedTraceResponsePricingPolicy, ): FixedTraceApprovedPricing { const approved = approvedResponsePricingPolicies.get(policy); - if (!approved) throw new Error('Fixed trace returned-model pricing policy is not evaluator approved'); + if (!approved) + throw new Error( + "Fixed trace returned-model pricing policy is not evaluator approved", + ); return approved; } export function fixedTraceResponsePricingPolicy( - expectedProvider: ModelProvider['id'], + expectedProvider: ModelProvider["id"], expectedModel: string, pricing: FixedTraceBudgetPricing & { readonly profileId: string }, ): FixedTraceResponsePricingPolicy { - const approved = FIXED_TRACE_APPROVED_PRICING.find((entry) => ( - entry.expectedProvider === expectedProvider - && entry.expectedModel === expectedModel - && entry.modelResolutionPolicy === fixedTraceModelResolutionPolicy(expectedProvider, expectedModel) - && sameApprovedPricing(entry, pricing) - )); - if (!approved) throw new Error('Fixed trace pricing profile is not evaluator approved'); + const approved = FIXED_TRACE_APPROVED_PRICING.find( + (entry) => + entry.expectedProvider === expectedProvider && + entry.expectedModel === expectedModel && + entry.modelResolutionPolicy === + fixedTraceModelResolutionPolicy(expectedProvider, expectedModel) && + sameApprovedPricing(entry, pricing), + ); + if (!approved) + throw new Error("Fixed trace pricing profile is not evaluator approved"); const policy = Object.freeze({ expectedProvider: approved.expectedProvider, expectedModel: approved.expectedModel, @@ -199,9 +229,11 @@ export function fixedTraceResponseUsesPricingPolicy( const approved = approvedResponsePricing(policy); if (response.provider !== policy.expectedProvider) return false; if (response.model === policy.expectedModel) return true; - return policy.modelResolutionPolicy === 'google_router_dated_revision_v1' - && approved.profileId === GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION - && isGoogleRouterModelRevision(response.model); + return ( + policy.modelResolutionPolicy === "google_router_dated_revision_v1" && + approved.profileId === GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION && + isGoogleRouterModelRevision(response.model) + ); } interface Reservation { @@ -219,11 +251,11 @@ interface BudgetedProviderBinding { interface BudgetedDelegateIdentity { readonly delegate: ModelProvider; - readonly id: ModelProvider['id']; - readonly capabilities: ModelProvider['capabilities']; - readonly prepare: ModelProvider['prepare']; - readonly respond: ModelProvider['respond']; - readonly deriveProviderToolReceipt?: ModelProvider['deriveProviderToolReceipt']; + readonly id: ModelProvider["id"]; + readonly capabilities: ModelProvider["capabilities"]; + readonly prepare: ModelProvider["prepare"]; + readonly respond: ModelProvider["respond"]; + readonly deriveProviderToolReceipt?: ModelProvider["deriveProviderToolReceipt"]; } // This is deliberately not an instance field or a public predicate. The @@ -232,10 +264,14 @@ interface BudgetedDelegateIdentity { // replacing the dispatch path. const budgetedProviderBindings = new WeakMap(); const exclusiveBudgetLeases = new WeakMap(); -const exclusiveCloneIdentities = new WeakMap(); +const exclusiveCloneIdentities = new WeakMap< + object, + BudgetedDelegateIdentity +>(); function deepFreeze(value: T): T { - if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; + if (typeof value !== "object" || value === null || Object.isFrozen(value)) + return value; for (const nested of Object.values(value)) deepFreeze(nested); return Object.freeze(value); } @@ -244,16 +280,25 @@ function snapshotPricing(pricing: T): T { return Object.freeze({ ...pricing }) as T; } -function snapshotDelegateIdentity(delegate: ModelProvider): BudgetedDelegateIdentity { +function snapshotDelegateIdentity( + delegate: ModelProvider, +): BudgetedDelegateIdentity { // Read all mutable delegate surface once. Lease cloning reuses this sealed // identity rather than re-reading a delegate getter after preflight. const id = delegate.id; - const capabilities = deepFreeze(structuredClone(delegate.capabilities)) as ModelProvider['capabilities']; + const capabilities = deepFreeze( + structuredClone(delegate.capabilities), + ) as ModelProvider["capabilities"]; const prepare = delegate.prepare; const respond = delegate.respond; const deriveProviderToolReceipt = delegate.deriveProviderToolReceipt; - if (typeof id !== 'string' || !id.trim() || typeof prepare !== 'function' || typeof respond !== 'function') { - throw new Error('Fixed trace budget delegate identity is invalid'); + if ( + typeof id !== "string" || + !id.trim() || + typeof prepare !== "function" || + typeof respond !== "function" + ) { + throw new Error("Fixed trace budget delegate identity is invalid"); } return Object.freeze({ delegate, @@ -265,43 +310,61 @@ function snapshotDelegateIdentity(delegate: ModelProvider): BudgetedDelegateIden }); } -function samePricing(left: FixedTraceBudgetPricing, right: FixedTraceBudgetPricing): boolean { - return left.inputUsdPerMillionTokens === right.inputUsdPerMillionTokens - && left.outputUsdPerMillionTokens === right.outputUsdPerMillionTokens - && left.cacheReadUsdPerMillionTokens === right.cacheReadUsdPerMillionTokens - && left.cacheWriteUsdPerMillionTokens === right.cacheWriteUsdPerMillionTokens - && left.cacheReadAccounting === right.cacheReadAccounting - && left.cacheWriteAccounting === right.cacheWriteAccounting - && left.source === right.source; +function samePricing( + left: FixedTraceBudgetPricing, + right: FixedTraceBudgetPricing, +): boolean { + return ( + left.inputUsdPerMillionTokens === right.inputUsdPerMillionTokens && + left.outputUsdPerMillionTokens === right.outputUsdPerMillionTokens && + left.cacheReadUsdPerMillionTokens === right.cacheReadUsdPerMillionTokens && + left.cacheWriteUsdPerMillionTokens === + right.cacheWriteUsdPerMillionTokens && + left.cacheReadAccounting === right.cacheReadAccounting && + left.cacheWriteAccounting === right.cacheWriteAccounting && + left.source === right.source + ); } function sameResponsePricingPolicy( left: FixedTraceResponsePricingPolicy, right: FixedTraceResponsePricingPolicy, ): boolean { - return left.expectedProvider === right.expectedProvider - && left.expectedModel === right.expectedModel - && left.pricingProfileId === right.pricingProfileId - && left.modelResolutionPolicy === right.modelResolutionPolicy; + return ( + left.expectedProvider === right.expectedProvider && + left.expectedModel === right.expectedModel && + left.pricingProfileId === right.pricingProfileId && + left.modelResolutionPolicy === right.modelResolutionPolicy + ); } -export function validateFixedTracePricing(pricing: FixedTraceBudgetPricing): void { +export function validateFixedTracePricing( + pricing: FixedTraceBudgetPricing, +): void { if ( - !Number.isFinite(pricing.inputUsdPerMillionTokens) - || pricing.inputUsdPerMillionTokens < 0 - || !Number.isFinite(pricing.outputUsdPerMillionTokens) - || pricing.outputUsdPerMillionTokens < 0 - || !pricing.source.trim() - ) throw new Error('Fixed trace budget pricing is invalid'); - for (const rate of [pricing.cacheReadUsdPerMillionTokens, pricing.cacheWriteUsdPerMillionTokens]) { - if (rate !== undefined && rate !== null && (!Number.isFinite(rate) || rate < 0)) { - throw new Error('Fixed trace cache pricing is invalid'); + !Number.isFinite(pricing.inputUsdPerMillionTokens) || + pricing.inputUsdPerMillionTokens < 0 || + !Number.isFinite(pricing.outputUsdPerMillionTokens) || + pricing.outputUsdPerMillionTokens < 0 || + !pricing.source.trim() + ) + throw new Error("Fixed trace budget pricing is invalid"); + for (const rate of [ + pricing.cacheReadUsdPerMillionTokens, + pricing.cacheWriteUsdPerMillionTokens, + ]) { + if ( + rate !== undefined && + rate !== null && + (!Number.isFinite(rate) || rate < 0) + ) { + throw new Error("Fixed trace cache pricing is invalid"); } } } function requestBytes(prepared: PreparedModelInvocation): number { - return Buffer.byteLength(JSON.stringify(prepared.providerRequest), 'utf8'); + return Buffer.byteLength(JSON.stringify(prepared.providerRequest), "utf8"); } /** @@ -316,41 +379,54 @@ export function fixedTraceEstimatedCostUsd( validateFixedTracePricing(pricing); const { inputTokens, outputTokens } = usage; if ( - !Number.isSafeInteger(inputTokens) - || inputTokens < 0 - || !Number.isSafeInteger(outputTokens) - || outputTokens < 0 - ) throw new Error('Fixed trace budget usage is invalid'); + !Number.isSafeInteger(inputTokens) || + inputTokens < 0 || + !Number.isSafeInteger(outputTokens) || + outputTokens < 0 + ) + throw new Error("Fixed trace budget usage is invalid"); const cacheReadTokens = usage.cacheReadTokens ?? 0; const cacheWriteTokens = usage.cacheWriteTokens ?? 0; if ( - !Number.isSafeInteger(cacheReadTokens) || cacheReadTokens < 0 - || !Number.isSafeInteger(cacheWriteTokens) || cacheWriteTokens < 0 - ) throw new Error('Fixed trace cache usage is invalid'); - const readAccounting = pricing.cacheReadAccounting ?? 'unsupported'; - const writeAccounting = pricing.cacheWriteAccounting ?? 'unsupported'; - if (cacheReadTokens > 0 && readAccounting === 'unsupported') throw new Error('Fixed trace cache read accounting is unavailable'); - if (cacheWriteTokens > 0 && writeAccounting === 'unsupported') throw new Error('Fixed trace cache write accounting is unavailable'); - if (readAccounting === 'subset' && cacheReadTokens > inputTokens) throw new Error('Fixed trace subset cache read usage is invalid'); + !Number.isSafeInteger(cacheReadTokens) || + cacheReadTokens < 0 || + !Number.isSafeInteger(cacheWriteTokens) || + cacheWriteTokens < 0 + ) + throw new Error("Fixed trace cache usage is invalid"); + const readAccounting = pricing.cacheReadAccounting ?? "unsupported"; + const writeAccounting = pricing.cacheWriteAccounting ?? "unsupported"; + if (cacheReadTokens > 0 && readAccounting === "unsupported") + throw new Error("Fixed trace cache read accounting is unavailable"); + if (cacheWriteTokens > 0 && writeAccounting === "unsupported") + throw new Error("Fixed trace cache write accounting is unavailable"); + if (readAccounting === "subset" && cacheReadTokens > inputTokens) + throw new Error("Fixed trace subset cache read usage is invalid"); // A subset read and additive write (Google's profile) is valid. Two subset // buckets must jointly fit the provider's normalized input total. - if (readAccounting === 'subset' && writeAccounting === 'subset' && cacheReadTokens + cacheWriteTokens > inputTokens) { - throw new Error('Fixed trace subset cache usage is invalid'); + if ( + readAccounting === "subset" && + writeAccounting === "subset" && + cacheReadTokens + cacheWriteTokens > inputTokens + ) { + throw new Error("Fixed trace subset cache usage is invalid"); } if (cacheReadTokens > 0 && pricing.cacheReadUsdPerMillionTokens == null) { - throw new Error('Fixed trace cache read pricing is unavailable'); + throw new Error("Fixed trace cache read pricing is unavailable"); } if (cacheWriteTokens > 0 && pricing.cacheWriteUsdPerMillionTokens == null) { - throw new Error('Fixed trace cache write pricing is unavailable'); + throw new Error("Fixed trace cache write pricing is unavailable"); } return ( - (inputTokens - - (readAccounting === 'subset' ? cacheReadTokens : 0) - - (writeAccounting === 'subset' ? cacheWriteTokens : 0)) * pricing.inputUsdPerMillionTokens - + outputTokens * pricing.outputUsdPerMillionTokens - + cacheReadTokens * (pricing.cacheReadUsdPerMillionTokens ?? 0) - + cacheWriteTokens * (pricing.cacheWriteUsdPerMillionTokens ?? 0) - ) / 1_000_000; + ((inputTokens - + (readAccounting === "subset" ? cacheReadTokens : 0) - + (writeAccounting === "subset" ? cacheWriteTokens : 0)) * + pricing.inputUsdPerMillionTokens + + outputTokens * pricing.outputUsdPerMillionTokens + + cacheReadTokens * (pricing.cacheReadUsdPerMillionTokens ?? 0) + + cacheWriteTokens * (pricing.cacheWriteUsdPerMillionTokens ?? 0)) / + 1_000_000 + ); } /** @@ -371,7 +447,7 @@ export class FixedTraceBudget { constructor(readonly softMaxUsd: number) { if (!Number.isFinite(softMaxUsd) || softMaxUsd <= 0) { - throw new RangeError('Fixed trace soft budget must be positive'); + throw new RangeError("Fixed trace soft budget must be positive"); } } @@ -384,18 +460,25 @@ export class FixedTraceBudget { validateFixedTracePricing(pricing); const exclusiveLease = exclusiveBudgetLeases.get(this); if (exclusiveLease !== undefined && lease !== exclusiveLease) { - throw new Error('Fixed trace budget is reserved for an exclusive diagnostic run'); + throw new Error( + "Fixed trace budget is reserved for an exclusive diagnostic run", + ); } if (!Number.isSafeInteger(maxOutputTokens) || maxOutputTokens < 1) { - throw new RangeError('Fixed trace output reserve must be a positive integer'); + throw new RangeError( + "Fixed trace output reserve must be a positive integer", + ); } if (this.exposureUnknown) { this.budgetRejectedCalls++; - throw new FixedTraceBudgetAdmissionError('budget_exposure_unknown', prepared); + throw new FixedTraceBudgetAdmissionError( + "budget_exposure_unknown", + prepared, + ); } if (this.admissionClosed) { this.budgetRejectedCalls++; - throw new FixedTraceBudgetAdmissionError('soft_limit_exceeded', prepared); + throw new FixedTraceBudgetAdmissionError("soft_limit_exceeded", prepared); } // Request bytes are a deliberately high token bound for the request. An // additive cache bucket is separately billable, so reserve that same @@ -403,16 +486,21 @@ export class FixedTraceBudget { // by inputTokens. This keeps the pre-dispatch reserve conservative under // the recorded, fingerprinted cache formula. const inputTokens = requestBytes(prepared); - const usd = fixedTraceEstimatedCostUsd({ - inputTokens, - outputTokens: maxOutputTokens, - cacheReadTokens: pricing.cacheReadAccounting === 'additive' ? inputTokens : 0, - cacheWriteTokens: pricing.cacheWriteAccounting === 'additive' ? inputTokens : 0, - }, pricing); + const usd = fixedTraceEstimatedCostUsd( + { + inputTokens, + outputTokens: maxOutputTokens, + cacheReadTokens: + pricing.cacheReadAccounting === "additive" ? inputTokens : 0, + cacheWriteTokens: + pricing.cacheWriteAccounting === "additive" ? inputTokens : 0, + }, + pricing, + ); if (this.accountedSpendUsd + this.reservedUsd + usd > this.softMaxUsd) { this.admissionClosed = true; this.budgetRejectedCalls++; - throw new FixedTraceBudgetAdmissionError('soft_limit_exceeded', prepared); + throw new FixedTraceBudgetAdmissionError("soft_limit_exceeded", prepared); } this.reservedUsd += usd; return { usd, active: true }; @@ -445,13 +533,16 @@ export class FixedTraceBudget { snapshot(): FixedTraceBudgetSnapshot { return Object.freeze({ - policy: 'soft_admission_target', + policy: "soft_admission_target", softMaxUsd: this.softMaxUsd, accountedSpendUsd: this.accountedSpendUsd, reservedUsd: this.reservedUsd, remainingUsd: this.exposureUnknown ? null - : Math.max(0, this.softMaxUsd - this.accountedSpendUsd - this.reservedUsd), + : Math.max( + 0, + this.softMaxUsd - this.accountedSpendUsd - this.reservedUsd, + ), dispatchedCalls: this.dispatchedCalls, completedCalls: this.completedCalls, budgetRejectedCalls: this.budgetRejectedCalls, @@ -461,7 +552,8 @@ export class FixedTraceBudget { } private requireActive(reservation: Reservation): void { - if (!reservation.active) throw new Error('Fixed trace budget reservation is inactive'); + if (!reservation.active) + throw new Error("Fixed trace budget reservation is inactive"); } private release(reservation: Reservation): void { @@ -473,9 +565,9 @@ export class FixedTraceBudget { /** Model-provider decorator that applies a shared budget at the dispatch edge. */ export class BudgetedFixedTraceProvider implements ModelProvider { - readonly id: ModelProvider['id']; - readonly capabilities: ModelProvider['capabilities']; - readonly deriveProviderToolReceipt?: ModelProvider['deriveProviderToolReceipt']; + readonly id: ModelProvider["id"]; + readonly capabilities: ModelProvider["capabilities"]; + readonly deriveProviderToolReceipt?: ModelProvider["deriveProviderToolReceipt"]; readonly #delegate: BudgetedDelegateIdentity; readonly #budget: FixedTraceBudget; @@ -491,17 +583,23 @@ export class BudgetedFixedTraceProvider implements ModelProvider { ) { const approvedPricing = approvedResponsePricing(responsePricingPolicy); if (!sameApprovedPricing(approvedPricing, pricing)) { - throw new Error('Fixed trace budget pricing does not match its evaluator-approved policy'); + throw new Error( + "Fixed trace budget pricing does not match its evaluator-approved policy", + ); } - const clonedIdentity = cloneIdentityToken === undefined - ? undefined - : exclusiveCloneIdentities.get(cloneIdentityToken); + const clonedIdentity = + cloneIdentityToken === undefined + ? undefined + : exclusiveCloneIdentities.get(cloneIdentityToken); if (cloneIdentityToken !== undefined && !clonedIdentity) { - throw new Error('Fixed trace budget clone identity is unavailable'); + throw new Error("Fixed trace budget clone identity is unavailable"); } - const delegateIdentity = clonedIdentity ?? snapshotDelegateIdentity(delegate); + const delegateIdentity = + clonedIdentity ?? snapshotDelegateIdentity(delegate); if (delegateIdentity.id !== responsePricingPolicy.expectedProvider) { - throw new Error('Fixed trace budget delegate identity does not match its pricing policy'); + throw new Error( + "Fixed trace budget delegate identity does not match its pricing policy", + ); } this.#delegate = delegateIdentity; this.#budget = budget; @@ -510,7 +608,8 @@ export class BudgetedFixedTraceProvider implements ModelProvider { this.id = delegateIdentity.id; this.capabilities = delegateIdentity.capabilities; if (delegateIdentity.deriveProviderToolReceipt) { - this.deriveProviderToolReceipt = delegateIdentity.deriveProviderToolReceipt; + this.deriveProviderToolReceipt = + delegateIdentity.deriveProviderToolReceipt; } budgetedProviderBindings.set(this, { budget, @@ -545,7 +644,12 @@ export class BudgetedFixedTraceProvider implements ModelProvider { ...options, beforeDispatch: async (prepared) => { this.assertPreparedIdentity(prepared); - reservation = this.#budget.reserve(prepared, request.maxOutputTokens, this.#pricing, lease ?? undefined); + reservation = this.#budget.reserve( + prepared, + request.maxOutputTokens, + this.#pricing, + lease ?? undefined, + ); try { await options.beforeDispatch?.(prepared); } catch (error) { @@ -557,16 +661,23 @@ export class BudgetedFixedTraceProvider implements ModelProvider { dispatchStarted = true; }, })) { - if (event.type === 'response_complete') { + if (event.type === "response_complete") { if (!reservation || !dispatchStarted) { - throw new Error('Fixed trace provider completed without dispatch admission'); + throw new Error( + "Fixed trace provider completed without dispatch admission", + ); } // The delegate still owns `event.response` and may mutate it when // the iterator resumes after this yield. One evaluator-owned frozen // snapshot is therefore the sole terminal response used for // approval, settlement, and the outward event. const response = deepFreeze(structuredClone(event.response)); - if (fixedTraceResponseUsesPricingPolicy(this.#responsePricingPolicy, response)) { + if ( + fixedTraceResponseUsesPricingPolicy( + this.#responsePricingPolicy, + response, + ) + ) { this.#budget.complete(reservation, response.usage, this.#pricing); } else { // Do not settle an unapproved returned identity at the requested @@ -576,7 +687,7 @@ export class BudgetedFixedTraceProvider implements ModelProvider { this.#budget.markExposureUnknown(reservation); } settled = true; - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; continue; } yield event; @@ -591,15 +702,20 @@ export class BudgetedFixedTraceProvider implements ModelProvider { private assertRequestIdentity(request: ModelRequest): void { if (request.model !== this.#responsePricingPolicy.expectedModel) { - throw new Error('Fixed trace budget request model does not match its pricing policy'); + throw new Error( + "Fixed trace budget request model does not match its pricing policy", + ); } } private assertPreparedIdentity(prepared: PreparedModelInvocation): void { if ( - prepared.provider !== this.id - || prepared.model !== this.#responsePricingPolicy.expectedModel - ) throw new Error('Fixed trace budget prepared invocation identity does not match its pricing policy'); + prepared.provider !== this.id || + prepared.model !== this.#responsePricingPolicy.expectedModel + ) + throw new Error( + "Fixed trace budget prepared invocation identity does not match its pricing policy", + ); } static cloneForExclusiveDiagnosticRun( @@ -607,7 +723,8 @@ export class BudgetedFixedTraceProvider implements ModelProvider { lease: object, ): BudgetedFixedTraceProvider { const binding = budgetedProviderBindings.get(source); - if (!binding) throw new Error('Fixed trace budget wrapper binding is unavailable'); + if (!binding) + throw new Error("Fixed trace budget wrapper binding is unavailable"); const cloneIdentityToken = Object.freeze({}); exclusiveCloneIdentities.set(cloneIdentityToken, binding.delegate); let clone: BudgetedFixedTraceProvider; @@ -623,14 +740,17 @@ export class BudgetedFixedTraceProvider implements ModelProvider { exclusiveCloneIdentities.delete(cloneIdentityToken); } const cloneBinding = budgetedProviderBindings.get(clone); - if (!cloneBinding) throw new Error('Fixed trace budget wrapper binding is unavailable'); + if (!cloneBinding) + throw new Error("Fixed trace budget wrapper binding is unavailable"); cloneBinding.lease = lease; return clone; } } -const budgetedFixedTraceProviderPrepare = BudgetedFixedTraceProvider.prototype.prepare; -const budgetedFixedTraceProviderRespond = BudgetedFixedTraceProvider.prototype.respond; +const budgetedFixedTraceProviderPrepare = + BudgetedFixedTraceProvider.prototype.prepare; +const budgetedFixedTraceProviderRespond = + BudgetedFixedTraceProvider.prototype.respond; Object.freeze(BudgetedFixedTraceProvider.prototype); Object.freeze(BudgetedFixedTraceProvider); @@ -647,15 +767,26 @@ export function isTrustedBudgetedFixedTraceProvider( ): boolean { const binding = budgetedProviderBindings.get(provider); if ( - binding?.budget !== budget - || !samePricing(binding.pricing, pricing) - || !sameResponsePricingPolicy(binding.responsePricingPolicy, responsePricingPolicy) - ) return false; - if (Object.getPrototypeOf(provider) !== BudgetedFixedTraceProvider.prototype) return false; - if ((provider as unknown as { constructor: unknown }).constructor !== BudgetedFixedTraceProvider) return false; - return Object.isFrozen(provider) - && provider.prepare === budgetedFixedTraceProviderPrepare - && provider.respond === budgetedFixedTraceProviderRespond; + binding?.budget !== budget || + !samePricing(binding.pricing, pricing) || + !sameResponsePricingPolicy( + binding.responsePricingPolicy, + responsePricingPolicy, + ) + ) + return false; + if (Object.getPrototypeOf(provider) !== BudgetedFixedTraceProvider.prototype) + return false; + if ( + (provider as unknown as { constructor: unknown }).constructor !== + BudgetedFixedTraceProvider + ) + return false; + return ( + Object.isFrozen(provider) && + provider.prepare === budgetedFixedTraceProviderPrepare && + provider.respond === budgetedFixedTraceProviderRespond + ); } export interface FixedTraceBudgetDiagnosticLease { @@ -675,38 +806,52 @@ export function claimFixedTraceBudgetDiagnosticLease( ): FixedTraceBudgetDiagnosticLease { const snapshot = budget.snapshot(); if ( - snapshot.accountedSpendUsd !== 0 - || snapshot.reservedUsd !== 0 - || snapshot.dispatchedCalls !== 0 - || snapshot.completedCalls !== 0 - || snapshot.budgetRejectedCalls !== 0 - || snapshot.admissionClosed - || snapshot.exposureUnknown - || exclusiveBudgetLeases.has(budget) - ) throw new Error('Fixed trace diagnostic budget must be pristine and exclusively claimed'); + snapshot.accountedSpendUsd !== 0 || + snapshot.reservedUsd !== 0 || + snapshot.dispatchedCalls !== 0 || + snapshot.completedCalls !== 0 || + snapshot.budgetRejectedCalls !== 0 || + snapshot.admissionClosed || + snapshot.exposureUnknown || + exclusiveBudgetLeases.has(budget) + ) + throw new Error( + "Fixed trace diagnostic budget must be pristine and exclusively claimed", + ); const lease = Object.freeze({}); const clones = new Map(); for (const provider of providers) { if (clones.has(provider)) continue; const binding = budgetedProviderBindings.get(provider); - if (!binding || !isTrustedBudgetedFixedTraceProvider( - provider, - budget, - binding.pricing, - binding.responsePricingPolicy, - )) { - throw new Error('Fixed trace diagnostic provider is not an authenticated budget wrapper'); + if ( + !binding || + !isTrustedBudgetedFixedTraceProvider( + provider, + budget, + binding.pricing, + binding.responsePricingPolicy, + ) + ) { + throw new Error( + "Fixed trace diagnostic provider is not an authenticated budget wrapper", + ); } - clones.set(provider, BudgetedFixedTraceProvider.cloneForExclusiveDiagnosticRun( - provider as BudgetedFixedTraceProvider, - lease, - )); + clones.set( + provider, + BudgetedFixedTraceProvider.cloneForExclusiveDiagnosticRun( + provider as BudgetedFixedTraceProvider, + lease, + ), + ); } const diagnosticLease = Object.freeze({ providerFor(provider: ModelProvider): BudgetedFixedTraceProvider { const clone = clones.get(provider); - if (!clone) throw new Error('Fixed trace diagnostic provider is missing from its exclusive lease'); + if (!clone) + throw new Error( + "Fixed trace diagnostic provider is missing from its exclusive lease", + ); return clone; }, }); @@ -714,16 +859,19 @@ export function claimFixedTraceBudgetDiagnosticLease( const sourceBinding = budgetedProviderBindings.get(source); const cloneBinding = budgetedProviderBindings.get(clone); if ( - !sourceBinding - || !cloneBinding - || cloneBinding.delegate !== sourceBinding.delegate - || !isTrustedBudgetedFixedTraceProvider( + !sourceBinding || + !cloneBinding || + cloneBinding.delegate !== sourceBinding.delegate || + !isTrustedBudgetedFixedTraceProvider( clone, budget, sourceBinding.pricing, sourceBinding.responsePricingPolicy, ) - ) throw new Error('Fixed trace diagnostic clone identity is not authenticated'); + ) + throw new Error( + "Fixed trace diagnostic clone identity is not authenticated", + ); } // Diagnostic plans can additionally validate their cloned stages here. The // callback has no asynchronous boundary and runs before the lease becomes diff --git a/server/src/addie/eval/fixed-trace-diagnostic-cli.ts b/server/src/addie/eval/fixed-trace-diagnostic-cli.ts index ad6b7694a5..970f6817a4 100644 --- a/server/src/addie/eval/fixed-trace-diagnostic-cli.ts +++ b/server/src/addie/eval/fixed-trace-diagnostic-cli.ts @@ -4,10 +4,12 @@ export interface FixedTraceDiagnosticCliArguments { suite?: string; softMaxUsd?: string; output?: string; + experimentPlan?: string; + trustedManifest?: string; validateOnly: boolean; } -const NAMES = new Set(['providers', 'architecture-arm', 'suite', 'soft-max-usd', 'output', 'validate-only']); +const NAMES = new Set(['providers', 'architecture-arm', 'suite', 'soft-max-usd', 'output', 'experiment-plan', 'trusted-manifest', 'validate-only']); /** Strict, side-effect-free parser for the diagnostic-only manual evaluator. */ export function parseFixedTraceDiagnosticCliArguments(values: readonly string[]): FixedTraceDiagnosticCliArguments { @@ -33,6 +35,8 @@ export function parseFixedTraceDiagnosticCliArguments(values: readonly string[]) suite: typeof seen.get('suite') === 'string' ? seen.get('suite') as string : undefined, softMaxUsd: typeof seen.get('soft-max-usd') === 'string' ? seen.get('soft-max-usd') as string : undefined, output: typeof seen.get('output') === 'string' ? seen.get('output') as string : undefined, + experimentPlan: typeof seen.get('experiment-plan') === 'string' ? seen.get('experiment-plan') as string : undefined, + trustedManifest: typeof seen.get('trusted-manifest') === 'string' ? seen.get('trusted-manifest') as string : undefined, validateOnly: seen.get('validate-only') === true || seen.get('validate-only') === 'true', }; } diff --git a/server/src/addie/eval/fixed-trace-diagnostic-run.ts b/server/src/addie/eval/fixed-trace-diagnostic-run.ts index b319cad9d8..1e29de092e 100644 --- a/server/src/addie/eval/fixed-trace-diagnostic-run.ts +++ b/server/src/addie/eval/fixed-trace-diagnostic-run.ts @@ -23,6 +23,8 @@ import { fixedTraceResponsePricingPolicy, isTrustedBudgetedFixedTraceProvider, } from './fixed-trace-budget.js'; +import { types } from 'node:util'; +import { snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js'; export interface FixedTraceDiagnosticProviderPlan { readonly name: string; @@ -64,6 +66,17 @@ function ownDataProperty(source: unknown, name: string, owner: string): unknown return descriptor.value; } +function assertClosedOwnDataRecord(source: unknown, fields: readonly string[], owner: string): void { + if (typeof source !== 'object' || source === null || types.isProxy(source) || Object.getPrototypeOf(source) !== Object.prototype) { + throw new Error(`Fixed trace diagnostic ${owner} must be a plain non-Proxy object`); + } + const keys = Reflect.ownKeys(source); + if (keys.length !== fields.length || keys.some((key) => typeof key !== 'string' || !fields.includes(key))) { + throw new Error(`Fixed trace diagnostic ${owner} must contain exactly its approved fields`); + } + for (const field of fields) ownDataProperty(source, field, owner); +} + const DIAGNOSTIC_PRICING_FIELDS = [ 'profileId', 'inputUsdPerMillionTokens', @@ -76,19 +89,7 @@ const DIAGNOSTIC_PRICING_FIELDS = [ ] as const; function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing { - const prototype = typeof pricing === 'object' && pricing !== null - ? Object.getPrototypeOf(pricing) - : null; - if ( - typeof pricing !== 'object' - || pricing === null - || (prototype !== Object.prototype && prototype !== null) - ) throw new Error(`Fixed trace diagnostic ${owner} must be a plain pricing object`); - const keys = Reflect.ownKeys(pricing); - if ( - keys.length !== DIAGNOSTIC_PRICING_FIELDS.length - || keys.some((key) => typeof key !== 'string' || !DIAGNOSTIC_PRICING_FIELDS.includes(key as typeof DIAGNOSTIC_PRICING_FIELDS[number])) - ) throw new Error(`Fixed trace diagnostic ${owner} must contain only approved pricing fields`); + assertClosedOwnDataRecord(pricing, DIAGNOSTIC_PRICING_FIELDS, owner); // Structured cloning calls nested getters. Copy each approved data // descriptor instead, so a price cannot change between validation and use. return Object.freeze({ @@ -106,6 +107,10 @@ function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing { function snapshotStageConfig(config: unknown, owner: string): FixedTraceProviderStageConfig { // Read each untrusted stage property exactly once. Later checks use only // this detached plain object, never a caller-controlled getter or proxy. + assertClosedOwnDataRecord(config, [ + 'provider', 'model', 'reasoningEffort', 'maxOutputTokens', 'timeoutMs', + 'maxIterations', 'transportRetries', 'samplingMode', 'temperature', 'pricing', + ], owner); const provider = ownDataProperty(config, 'provider', owner); const model = ownDataProperty(config, 'model', owner); const reasoningEffort = ownDataProperty(config, 'reasoningEffort', owner); @@ -133,12 +138,7 @@ function snapshotStageConfig(config: unknown, owner: string): FixedTraceProvider function snapshotBaseConfig( config: FixedTraceDiagnosticArtifactOptions['baseConfig'], ): FixedTraceDiagnosticArtifactOptions['baseConfig'] { - const { traceSuite, toolDefinitions, ...serializable } = config; - return Object.freeze({ - ...structuredClone(serializable), - traceSuite: deepFreeze(structuredClone(traceSuite)), - toolDefinitions: deepFreeze(structuredClone(toolDefinitions)), - }); + return snapshotFixedTraceJson(config, 'fixed trace diagnostic base config') as FixedTraceDiagnosticArtifactOptions['baseConfig']; } function snapshotPlans( @@ -148,6 +148,19 @@ function snapshotPlans( if (!Array.isArray(suppliedPlans) || suppliedPlans.length === 0) { throw new Error('Fixed trace diagnostic run requires one or more provider plans'); } + if (types.isProxy(suppliedPlans) || Object.getPrototypeOf(suppliedPlans) !== Array.prototype || Object.getOwnPropertySymbols(suppliedPlans).length !== 0) { + throw new Error('Fixed trace diagnostic provider plans must be a plain non-Proxy array'); + } + const planDescriptors = Object.getOwnPropertyDescriptors(suppliedPlans); + for (const key of Object.keys(planDescriptors)) { + if (key === 'length') continue; + if (!/^(0|[1-9][0-9]*)$/.test(key) || !('value' in planDescriptors[key]!) || !planDescriptors[key]!.enumerable) { + throw new Error('Fixed trace diagnostic provider plans contain an accessor or extra property'); + } + } + for (const [index, suppliedPlan] of suppliedPlans.entries()) { + assertClosedOwnDataRecord(suppliedPlan, ['name', 'router', 'generation'], `provider plan ${index}`); + } const plans = Object.freeze(suppliedPlans.map((suppliedPlan, index) => Object.freeze({ // Do not validate while reading: a plan accessor must not be able to // return one identity for validation and another for execution. diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts new file mode 100644 index 0000000000..63e9ae87f5 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -0,0 +1,1305 @@ +import { createHash } from "node:crypto"; +import { CLAUDE_PRICING_VERSION } from "../claude-pricing.js"; +import { + GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + OPENAI_GPT_5_6_LUNA_PRICING, +} from "../model-cost-pricing.js"; +import { ANTHROPIC_PROVIDER_CAPABILITIES } from "../model-providers/anthropic-provider.js"; +import { + GOOGLE_GENERATE_CONTENT_CAPABILITIES, + GOOGLE_ROUTER_MODEL, +} from "../model-providers/google-generate-content-provider.js"; +import type { + ModelProviderId, + ModelReasoningEffort, +} from "../model-providers/model-provider.js"; +import { + OPENAI_RESPONSES_CAPABILITIES, + OPENAI_ROUTER_MODEL, +} from "../model-providers/openai-responses-provider.js"; +import { ANTHROPIC_ROUTER_CAPABILITIES } from "../model-providers/anthropic-router-provider.js"; +import { + decideFixedTraceHybridRoute, + fixedTraceHybridPolicy, + type FixedTraceArchitectureArmId, +} from "./fixed-trace-architecture.js"; +import { + fixedTraceEstimatedCostUsd, + validateFixedTracePricing, + type FixedTraceBudgetPricing, +} from "./fixed-trace-budget.js"; +import { + FIXED_TRACE_PARTITION_MANIFEST, + assertFixedTracePartitionManifest, +} from "./fixed-trace-partition.js"; +import { snapshotFixedTraceJson } from "./fixed-trace-safe-snapshot.js"; +import { FIXED_TRACE_CORPUS } from "./fixed-trace-suite.js"; + +export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = + "addie-fixed-trace-evaluation-protocol-v3" as const; + +export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ + version: "addie-fixed-trace-confirmatory-power-v2", + familywiseAlpha: 0.025, + hypotheses: Object.freeze([ + Object.freeze({ + id: "H1-superiority", + comparison: "locked-pipeline-candidate vs locked-pipeline-comparator", + endpoint: "two-judge blinded quality success rate", + direction: "greater", + marginPercentagePoints: 0, + alternativeDifferencePercentagePoints: 5, + holmOneSidedAlpha: 0.0125, + exactTest: "exact_conditional_mcnemar_zero_margin_only", + }), + Object.freeze({ + id: "H2-quality-non-inferiority-for-lower-cost-pipeline", + comparison: + "lower-metered-cost locked pipeline quality vs locked-pipeline-comparator quality", + endpoint: "two-judge blinded quality success rate", + direction: "not_less_than", + marginPercentagePoints: -3, + alternativeDifferencePercentagePoints: 0, + holmOneSidedAlpha: 0.025, + exactTest: + "unavailable_pending_independent_Lloyd_Moldovan_score_statistic_E_plus_M_exact_unconditional_noninferiority_implementation_and_type_I_error_validation", + sensitivityOnly: + "Sidik_exact_CI_or_p_value_after_primary_Lloyd_Moldovan_verification", + }), + ]), + test: "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method", + bootstrap: "grouped_stratified_case_level_bootstrap", + exclusionRule: "hard_failures_and_missing_evidence_remain_in_denominator", + repetitionsCountAsIndependentCases: false, + superiorityRequiredIndependentEvaluableCases: 3_803, + nonInferiorityRequiredIndependentEvaluableCases: 10_562, + requiredIndependentEvaluableCases: 10_562, + targetPower: 0.8, + planningAlternative: + "H1: +5pp over zero; H2: 0pp, three points above the -3pp NI margin", + conservativeDiscordanceVarianceUpperBound: 1, + worstCaseUpperBoundsNotFinalN: true, + finalNReductionRule: + "only_sealed_never_reused_sizing_pilot_or_predeclared_blinded_arm_invariant_discordance_only_upward_internal_pilot_with_enumerated_adaptive_exact_type_I_error", + externalFinalN: null, + externalFinalStatus: + "unavailable_pending_fingerprinted_exact_paired_discordance_power_result", +} as const); + +/** + * This is the complete schema of the final statistical admission. Values that + * require independent custody are null, which is an executable refusal—not a + * prose promise. The sizing pilot is held out and may never be reused in the + * one-time final; repeated/template-related observations cluster by episode. + */ +export const FIXED_TRACE_CONFIRMATORY_ADMISSION = Object.freeze({ + status: "not_admitted_missing_fingerprinted_statistical_protocol", + reasons: Object.freeze([ + "external_final_pack_unavailable", + "held_out_sizing_pilot_and_conservative_discordance_bound_unavailable", + "independently_verified_Lloyd_Moldovan_E_plus_M_exact_unconditional_noninferiority_test_and_power_method_unavailable", + "candidate_comparator_arm_identity_unavailable", + "judge_calibration_must_be_separate_or_cross_fitted", + "privileged_evaluator_signer_and_durable_ledger_boundary_unavailable", + ]), + holm: Object.freeze({ + K: 2, + oneSidedFamilyAlpha: 0.025, + orderedAlphas: Object.freeze([0.0125, 0.025]), + }), + unitOfAnalysis: "unique_conversation_user_episode", + repeatedAndTemplateRelatedObservationRule: + "cluster_by_conversation_user_episode; repetitions_never_increase_N", + sizingPilot: Object.freeze({ + status: "unavailable", + heldOutFromFinal: true, + reusableInFinal: false, + conservativeDiscordanceUpperBound: null, + digest: null, + }), + judgeCalibration: Object.freeze({ + status: "unavailable", + allowedRelationshipToScoredDevelopment: "separate_or_cross_fitted_only", + digest: null, + }), + finalProtocolFingerprint: null, + externalPackDigest: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, +} as const); + +export type FixedTraceProtocolPhaseId = + | "stage_0_preflight_calibration" + | "stage_1_smoke" + | "stage_2_router_screen" + | "stage_2_oracle_generator_screen" + | "stage_3_architecture" + | "stage_4_tuning" + | "stage_5_external_final" + | "stage_6_canary"; +export type FixedTraceProtocolStageRole = + "router" | "generation" | "judge" | "simulator"; +export type FixedTraceProtocolAdmission = + | "admitted_diagnostic" + | "not_admitted_common_tool_universe" + | "not_admitted_architecture" + | "not_evaluable_no_treatment_contrast" + | "not_admitted_external_final" + | "not_admitted_canary"; + +export interface FixedTraceProtocolPricingProfile extends FixedTraceBudgetPricing { + readonly provider: ModelProviderId; + readonly model: string; + readonly version: string; +} + +export const FIXED_TRACE_PROTOCOL_PRICING = Object.freeze([ + Object.freeze({ + provider: "anthropic" as const, + model: "claude-haiku-4-5", + version: CLAUDE_PRICING_VERSION, + profileId: `${CLAUDE_PRICING_VERSION}:claude-haiku-4-5`, + inputUsdPerMillionTokens: 1, + outputUsdPerMillionTokens: 5, + cacheReadUsdPerMillionTokens: 0.1, + cacheWriteUsdPerMillionTokens: 1.25, + cacheReadAccounting: "additive" as const, + cacheWriteAccounting: "additive" as const, + source: "Repository Anthropic reviewed pricing table, August 2026.", + }), + Object.freeze({ + provider: "anthropic" as const, + model: "claude-sonnet-5", + version: CLAUDE_PRICING_VERSION, + profileId: `${CLAUDE_PRICING_VERSION}:claude-sonnet-5`, + inputUsdPerMillionTokens: 3, + outputUsdPerMillionTokens: 15, + cacheReadUsdPerMillionTokens: 0.3, + cacheWriteUsdPerMillionTokens: 3.75, + cacheReadAccounting: "additive" as const, + cacheWriteAccounting: "additive" as const, + source: "Repository Anthropic reviewed pricing table, August 2026.", + }), + Object.freeze({ + provider: "openai" as const, + model: OPENAI_ROUTER_MODEL, + version: OPENAI_GPT_5_6_LUNA_PRICING.profileId, + ...OPENAI_GPT_5_6_LUNA_PRICING, + }), + Object.freeze({ + provider: "google" as const, + model: GOOGLE_ROUTER_MODEL, + version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + profileId: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, + inputUsdPerMillionTokens: 0.75, + outputUsdPerMillionTokens: 3.75, + cacheReadUsdPerMillionTokens: 0.075, + cacheWriteUsdPerMillionTokens: 0.75, + cacheReadAccounting: "subset" as const, + cacheWriteAccounting: "additive" as const, + source: + "Repository Google Gemini 3.7 Flash pricing pin through 2026-12-31.", + }), +] satisfies readonly FixedTraceProtocolPricingProfile[]); + +export interface FixedTraceAdmittedCell { + readonly id: string; + readonly role: "router" | "generation"; + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly pricingProfileId: string; + readonly adapterCapabilitySource: string; +} +const efforts = (values: readonly ModelReasoningEffort[]) => + values.length ? values : ["provider_default" as const]; +const priceId = (provider: ModelProviderId, model: string) => { + const profile = FIXED_TRACE_PROTOCOL_PRICING.find( + (entry) => entry.provider === provider && entry.model === model, + ); + if (!profile) + throw new Error(`No immutable price for admitted ${provider}/${model}`); + return profile.profileId; +}; +const cells = ( + role: "router" | "generation", + provider: ModelProviderId, + model: string, + values: readonly ModelReasoningEffort[], + source: string, +) => + efforts(values).map((effort) => + Object.freeze({ + id: `${role}:${provider}:${model}:${effort}`, + role, + provider, + model, + effort, + pricingProfileId: priceId(provider, model), + adapterCapabilitySource: source, + }), + ); + +/** Derived only from reviewed exported adapter capabilities and immutable prices. */ +export const FIXED_TRACE_ADMITTED_CELLS: readonly FixedTraceAdmittedCell[] = + Object.freeze([ + ...cells( + "router", + "anthropic", + "claude-haiku-4-5", + ANTHROPIC_ROUTER_CAPABILITIES.reasoningEfforts, + "ANTHROPIC_ROUTER_CAPABILITIES", + ), + ...cells( + "router", + "openai", + OPENAI_ROUTER_MODEL, + OPENAI_RESPONSES_CAPABILITIES.reasoningEfforts, + "OPENAI_RESPONSES_CAPABILITIES", + ), + ...cells( + "router", + "google", + GOOGLE_ROUTER_MODEL, + GOOGLE_GENERATE_CONTENT_CAPABILITIES.reasoningEfforts, + "GOOGLE_GENERATE_CONTENT_CAPABILITIES", + ), + ...cells( + "generation", + "anthropic", + "claude-haiku-4-5", + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts.filter( + (effort) => effort === "provider_default", + ), + "ANTHROPIC_PROVIDER_CAPABILITIES", + ), + ...cells( + "generation", + "anthropic", + "claude-sonnet-5", + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts.filter( + (effort) => effort === "provider_default", + ), + "ANTHROPIC_PROVIDER_CAPABILITIES", + ), + ...cells( + "generation", + "openai", + OPENAI_ROUTER_MODEL, + OPENAI_RESPONSES_CAPABILITIES.reasoningEfforts, + "OPENAI_RESPONSES_CAPABILITIES", + ), + ...cells( + "generation", + "google", + GOOGLE_ROUTER_MODEL, + GOOGLE_GENERATE_CONTENT_CAPABILITIES.reasoningEfforts, + "GOOGLE_GENERATE_CONTENT_CAPABILITIES", + ), + ]); +export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ + Object.freeze({ + provider: "openai", + model: "gpt-5.6-terra", + dispatchable: false, + trustedPrice: null, + }), + Object.freeze({ + provider: "openai", + model: "gpt-5.6-sol", + dispatchable: false, + trustedPrice: null, + }), +]); + +/** + * This is the first *potentially* paid activity after credential-free + * admission. It is component-only: no semantic judges, no pipeline or + * architecture comparison, no execution authorization. + */ +export const FIXED_TRACE_COMPONENT_SMOKE_PLAN = Object.freeze({ + status: "not_admitted_pending_credential_free_admission", + cases: 8, + repetitions: 1, + routerCells: 10, + generationCells: 11, + totalComponentCells: 21, + maxRouterInvocationsPerCase: 1, + maxGenerationInvocationsPerCase: 2, + llmJudging: "none", + architectureClaim: "none", + providerCeilingUsd: 5, + authorization: "none", +}); + +/** Planning-only cardinalities; this does not schedule confirmation. */ +export const FIXED_TRACE_ARCHITECTURE_CELL_TRUTH = Object.freeze({ + routerCells: 10, + generationCells: 11, + directCombinations: 11, + twoStageCombinations: 110, + hybridCombinations: 110, + totalArchitectureCombinations: 231, + potentiallyLlmJudgeableProviderMatchedCombinations: 97, + mixedProviderCombinationsRequiringHumanOrFourthProvider: 134, +}); + +export const FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT = sha256( + FIXED_TRACE_ADMITTED_CELLS.map( + ({ id, role, provider, model, effort, pricingProfileId }) => ({ + id, + role, + provider, + model, + effort, + pricingProfileId, + }), + ), +); + +/** USD is operational evidence, never a percentage-point quality hypothesis. */ +export const FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE = Object.freeze({ + status: "not_admitted_pending_complete_trusted_usage_and_predeclared_economic_margin", + endpoint: "paired_metered_USD_cost_and_latency_reliability", + requiredEvidence: + "complete_trusted_usage_pricing_cost_for_every_dispatched_and_failed_timeout_unknown_exposure_invocation", + qualityHypothesisRelationship: + "separate_from_H2_quality_noninferiority", + binaryPercentagePointHypothesis: false, +}); + +/** No judge can score a finalist until this evaluator-custodied record exists. */ +export const FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS = Object.freeze([ + Object.freeze({ + provider: "anthropic" as const, + model: "claude-haiku-4-5", + effort: "provider_default" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), + Object.freeze({ + provider: "openai" as const, + model: OPENAI_ROUTER_MODEL, + effort: "none" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), + Object.freeze({ + provider: "google" as const, + model: GOOGLE_ROUTER_MODEL, + effort: "provider_default" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), +]); + +export interface FixedTraceProtocolStage { + readonly role: FixedTraceProtocolStageRole; + readonly cellId: string | null; + readonly maxInvocationsPerCase: number; + readonly maxInputTokensPerInvocation: number; + readonly maxOutputTokensPerInvocation: number; + readonly timeoutMs: number; + readonly retries: 0; + readonly cacheMode: "disabled"; + readonly sampling: "provider_no_sampling_control"; + /** No post-terminal invocation is expected; every eligible omission is a failure. */ + readonly invocationLifecycle: + | "always_eligible; dispatched_completed_terminal_usage_cost_recorded" + | "eligible_while_prior_tool_loop_is_nonterminal; post_terminal_not_eligible; eligible_omission_is_failure" + | "eligible_after_complete_candidate; candidate_hard_failure_remains_denominator"; +} +export interface FixedTraceProtocolArm { + readonly id: string; + readonly architecture: FixedTraceArchitectureArmId | "none"; + readonly admission: FixedTraceProtocolAdmission; + readonly selectedToolSubset: "architecture_derived_presented_subset"; + readonly stages: readonly FixedTraceProtocolStage[]; + readonly conditionalCalls?: { + readonly localTerminalCases: "exact_harmless_only"; + readonly fallbackRouterCallsPerNonlocalCase: 1; + readonly worstCaseRouterCalls: number; + }; +} +export interface FixedTraceProtocolPhase { + readonly id: FixedTraceProtocolPhaseId; + readonly caseSet: "development" | "tuning" | "external_unavailable"; + readonly uniqueCases: number | null; + readonly repetitions: number; + readonly selectionUse: + | "calibration" + | "adaptive_screening" + | "architecture_selection" + | "diagnostic_tuning" + | "confirmatory_unavailable" + | "default_off_canary_unavailable"; + readonly arms: readonly FixedTraceProtocolArm[]; +} +export interface FixedTraceEvaluationProtocol { + readonly version: typeof FIXED_TRACE_EVALUATION_PROTOCOL_VERSION; + readonly id: string; + readonly baseCapabilityUniverse: "one_authenticated_base_registry_schema_receipt_set"; + readonly phases: readonly FixedTraceProtocolPhase[]; + readonly adaptiveRule: { + readonly smokeCases: 8; + readonly developmentCases: 46; + readonly tuningCases: 36; + readonly deterministicElimination: readonly string[]; + readonly selection: "predeclared_pareto_successive_halving"; + readonly repeats: "stability_only_not_new_cases"; + }; + readonly finalProtocol: { + readonly status: "unavailable"; + readonly familywiseAlpha: 0.025; + readonly hypothesisIds: readonly [ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]; + readonly endpoint: "two-judge blinded quality success rate"; + readonly externalPackDigest: null; + readonly externalN: null; + readonly candidatePipelineId: null; + readonly comparatorPipelineId: null; + readonly architectureArmId: null; + readonly pairedTest: "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method"; + readonly bootstrap: "grouped_stratified_case_level_bootstrap"; + readonly exclusions: "hard_failures_and_missing_evidence_remain_in_denominator"; + readonly fingerprint: null; + readonly powerResult: null; + }; +} + +const stage = ( + role: FixedTraceProtocolStageRole, + cellId: string | null, + maxInvocationsPerCase: number, + maxInputTokensPerInvocation: number, + maxOutputTokensPerInvocation: number, +): FixedTraceProtocolStage => + Object.freeze({ + role, + cellId, + maxInvocationsPerCase, + maxInputTokensPerInvocation, + maxOutputTokensPerInvocation, + timeoutMs: 120_000, + retries: 0, + cacheMode: "disabled", + sampling: "provider_no_sampling_control", + invocationLifecycle: + role === "generation" && maxInvocationsPerCase > 1 + ? "eligible_while_prior_tool_loop_is_nonterminal; post_terminal_not_eligible; eligible_omission_is_failure" + : role === "judge" + ? "eligible_after_complete_candidate; candidate_hard_failure_remains_denominator" + : "always_eligible; dispatched_completed_terminal_usage_cost_recorded", + }); +const routerCell = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "router:anthropic:claude-haiku-4-5:provider_default", +)!; +const generatorCell = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:anthropic:claude-sonnet-5:provider_default", +)!; +const judgeCells = Object.freeze([ + FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:openai:gpt-5.6-luna:none", + )!, + FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:google:gemini-3.7-flash:provider_default", + )!, +]); +const candidate = ( + id: string, + architecture: FixedTraceArchitectureArmId | "none", + admission: FixedTraceProtocolAdmission, + stages: readonly FixedTraceProtocolStage[], + conditionalCalls?: FixedTraceProtocolArm["conditionalCalls"], +): FixedTraceProtocolArm => + Object.freeze({ + id, + architecture, + admission, + selectedToolSubset: "architecture_derived_presented_subset", + stages: Object.freeze(stages), + ...(conditionalCalls ? { conditionalCalls } : {}), + }); + +export interface FixedTraceHybridContrastPreflight { + readonly phase: "development" | "tuning"; + readonly totalCases: number; + readonly localTerminalCases: number; + readonly routedCases: number; + readonly minimumLocalTerminalCases: 1; + readonly minimumRoutedCases: 1; + readonly evaluable: boolean; + readonly blocker: "no_hybrid_treatment_contrast" | null; +} + +/** + * Positivity is measured only against the advertised corpus. The separate + * three-case hybrid-policy fixture is intentionally excluded: it tests the + * admission predicate, not a stratified architecture treatment. + */ +export function fixedTraceHybridContrastPreflight( + phase: "development" | "tuning", +): FixedTraceHybridContrastPreflight { + const traces = FIXED_TRACE_CORPUS.filter((trace) => trace.phase === phase); + const localTerminalCases = traces.filter( + (trace) => + decideFixedTraceHybridRoute({ + message: trace.request.message, + source: trace.request.source, + isAdmin: trace.request.isAdmin, + isThread: (trace.request.threadContext?.length ?? 0) > 0, + channelPrivacy: trace.request.channelPrivacy, + policy: fixedTraceHybridPolicy(), + }).mode === "local_terminal", + ).length; + const routedCases = traces.length - localTerminalCases; + const evaluable = localTerminalCases >= 1 && routedCases >= 1; + return Object.freeze({ + phase, + totalCases: traces.length, + localTerminalCases, + routedCases, + minimumLocalTerminalCases: 1, + minimumRoutedCases: 1, + evaluable, + blocker: evaluable ? null : "no_hybrid_treatment_contrast", + }); +} + +export const FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT = Object.freeze([ + fixedTraceHybridContrastPreflight("development"), + fixedTraceHybridContrastPreflight("tuning"), +]); + +export const FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL = Object.freeze({ + id: "fixed-trace-architecture-ablation-v2", + fixed: Object.freeze([ + "cases_and_order", + "locked_generator_finalist", + "one_authenticated_base_registry_schema_receipt_set", + "rules_prompts_simulator_receipts", + "limits_retries_cache_sampling", + "two_calibrated_blinded_provider_excluding_judges", + "all_planned_failures_denominator", + ]), + varied: "architecture_derived_tool_selection_and_presented_subset_only", +}); + +export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = + Object.freeze({ + version: FIXED_TRACE_EVALUATION_PROTOCOL_VERSION, + id: "addie-fixed-trace-adaptive-plan-v2", + baseCapabilityUniverse: + "one_authenticated_base_registry_schema_receipt_set", + adaptiveRule: Object.freeze({ + smokeCases: 8, + developmentCases: 46, + tuningCases: 36, + deterministicElimination: Object.freeze([ + "identity_or_pricing_mismatch", + "unauthorized_or_incorrect_mutation", + "malformed_empty_or_truncated_output", + "tool_loop_or_iteration_boundary", + "timeout_or_provider_error", + "missing_usage_or_ledger_mismatch", + "privacy_violation", + ]), + selection: "predeclared_pareto_successive_halving", + repeats: "stability_only_not_new_cases", + }), + finalProtocol: Object.freeze({ + status: "unavailable", + familywiseAlpha: 0.025, + hypothesisIds: Object.freeze([ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]) as readonly [ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ], + endpoint: "two-judge blinded quality success rate", + externalPackDigest: null, + externalN: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, + pairedTest: + "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method", + bootstrap: "grouped_stratified_case_level_bootstrap", + exclusions: "hard_failures_and_missing_evidence_remain_in_denominator", + fingerprint: null, + powerResult: null, + }), + phases: Object.freeze([ + Object.freeze({ + id: "stage_0_preflight_calibration", + caseSet: "development", + uniqueCases: 8, + repetitions: 1, + selectionUse: "calibration", + arms: Object.freeze([]), + }), + Object.freeze({ + id: "stage_1_smoke", + caseSet: "development", + uniqueCases: 8, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.map((cell) => + candidate(`smoke-${cell.id}`, "none", "admitted_diagnostic", [ + stage( + cell.role, + cell.id, + cell.role === "router" ? 1 : 2, + cell.role === "router" ? 4_096 : 16_384, + cell.role === "router" ? 300 : 900, + ), + ]), + ), + ), + }), + Object.freeze({ + id: "stage_2_router_screen", + caseSet: "development", + uniqueCases: 46, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.role === "router", + ).map((cell) => + candidate( + `router-screen-${cell.id}`, + "two_stage_llm_router", + "admitted_diagnostic", + [stage("router", cell.id, 1, 4_096, 300)], + ), + ), + ), + }), + Object.freeze({ + id: "stage_2_oracle_generator_screen", + caseSet: "development", + uniqueCases: 46, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.role === "generation", + ).map((cell) => + candidate( + `generator-screen-${cell.id}`, + "oracle_route_diagnostic", + "admitted_diagnostic", + [stage("generation", cell.id, 12, 16_384, 900)], + ), + ), + ), + }), + Object.freeze({ + id: "stage_3_architecture", + caseSet: "development", + uniqueCases: 46, + repetitions: 3, + selectionUse: "architecture_selection", + arms: Object.freeze([ + candidate( + "routed-locked-finalist", + "two_stage_llm_router", + "not_admitted_common_tool_universe", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + stage("judge", judgeCells[0].id, 1, 16_384, 300), + stage("judge", judgeCells[1].id, 1, 16_384, 300), + ], + ), + candidate( + "hybrid-locked-finalist", + "deterministic_policy_llm_fallback_hybrid", + "not_evaluable_no_treatment_contrast", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + stage("judge", judgeCells[0].id, 1, 16_384, 300), + stage("judge", judgeCells[1].id, 1, 16_384, 300), + ], + { + localTerminalCases: "exact_harmless_only", + fallbackRouterCallsPerNonlocalCase: 1, + worstCaseRouterCalls: 46 * 3, + }, + ), + candidate( + "direct-locked-finalist", + "direct_generation", + "not_admitted_architecture", + [stage("generation", generatorCell.id, 12, 16_384, 900)], + ), + ]), + }), + Object.freeze({ + id: "stage_4_tuning", + caseSet: "tuning", + uniqueCases: 36, + repetitions: 1, + selectionUse: "diagnostic_tuning", + arms: Object.freeze([ + candidate( + "tuning-locked-pipeline", + "two_stage_llm_router", + "admitted_diagnostic", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + ], + ), + ]), + }), + Object.freeze({ + id: "stage_5_external_final", + caseSet: "external_unavailable", + uniqueCases: null, + repetitions: 1, + selectionUse: "confirmatory_unavailable", + arms: Object.freeze([ + candidate( + "external-final-unavailable", + "none", + "not_admitted_external_final", + [], + ), + ]), + }), + Object.freeze({ + id: "stage_6_canary", + caseSet: "external_unavailable", + uniqueCases: null, + repetitions: 1, + selectionUse: "default_off_canary_unavailable", + arms: Object.freeze([ + candidate("canary-unavailable", "none", "not_admitted_canary", []), + ]), + }), + ]), + }); + +export interface FixedTraceStageCeiling { + phaseId: FixedTraceProtocolPhaseId; + armId: string; + role: FixedTraceProtocolStageRole; + calls: number; + ceilingUsd: number; +} +export interface FixedTraceProtocolEstimate { + dispatchable: false; + approvalCeilingUsd: null; + stages: readonly FixedTraceStageCeiling[]; + candidateCeilingUsd: number; + judgeCeilingUsd: number; + simulatorCeilingUsd: 0; + failedTimeoutUnknownExposureCeilingUsd: number; + contingencyUsd: number; + totalCeilingUsd: number; + componentSmokeCeilingUsd: number; + hybridWorstCaseRouterCalls: 138; + hybridWorstCaseRouterCeilingUsd: number; + armCallAccounting: readonly FixedTraceArchitectureArmCallAccounting[]; + externalFinalN: null; +} +export interface FixedTraceArchitectureArmCallAccounting { + readonly armId: string; + readonly admission: FixedTraceProtocolAdmission; + readonly evaluable: boolean; + readonly localTerminalCases: number; + readonly routedCases: number; + readonly routerCalls: number; + readonly generationCalls: number; + readonly routerCeilingUsd: number; + readonly generationCeilingUsd: number; +} +export interface FixedTraceScreeningResult { + readonly cellId: string; + readonly role: "router" | "generation"; + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly configFingerprint: string; + readonly safetyFailures: number; + readonly identityFailures: number; + readonly malformedFailures: number; + readonly toolLoopFailures: number; + readonly reliabilityFailures: number; + readonly latencyMs: number; + readonly costUsd: number; +} +/** Pure, predeclared elimination/halving rule; repetitions estimate stability only. */ +export function selectFixedTraceScreeningSurvivors( + results: readonly FixedTraceScreeningResult[], +): readonly string[] { + const snapshot = snapshotFixedTraceJson( + results, + "fixed-trace screening results", + ) as readonly FixedTraceScreeningResult[]; + const required = new Map( + FIXED_TRACE_ADMITTED_CELLS.map((cell) => [cell.id, cell]), + ); + const seen = new Set(); + if (snapshot.length !== required.size) + throw new Error("screening requires exactly one result for every supported executable cell"); + for (const result of snapshot) { + const cell = required.get(result.cellId); + if ( + !cell || + seen.has(result.cellId) || + result.role !== cell.role || + result.provider !== cell.provider || + result.model !== cell.model || + result.effort !== cell.effort || + result.configFingerprint !== FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT + ) + throw new Error("screening result has unknown, duplicate, or mismatched canonical cell identity"); + if ( + Object.keys(result).sort().join(",") !== + "cellId,configFingerprint,costUsd,effort,identityFailures,latencyMs,malformedFailures,model,provider,reliabilityFailures,role,safetyFailures,toolLoopFailures" + ) + throw new Error("screening result has extra or missing fields"); + if ( + [ + result.safetyFailures, + result.identityFailures, + result.malformedFailures, + result.toolLoopFailures, + result.reliabilityFailures, + result.latencyMs, + result.costUsd, + ].some((value) => !Number.isFinite(value) || value < 0) + ) + throw new Error("screening result has invalid metrics"); + seen.add(result.cellId); + } + const eligible = snapshot.filter( + (result) => + result.safetyFailures === 0 && + result.identityFailures === 0 && + result.malformedFailures === 0 && + result.toolLoopFailures === 0, + ); + return Object.freeze( + [...eligible] + .sort( + (left, right) => + left.reliabilityFailures - right.reliabilityFailures || + left.costUsd - right.costUsd || + left.latencyMs - right.latencyMs || + left.cellId.localeCompare(right.cellId), + ) + .slice(0, Math.max(1, Math.ceil(eligible.length / 2))) + .map((result) => result.cellId), + ); +} +function sha256(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(value), "utf8") + .digest("hex"); +} +export function fixedTraceEvaluationProtocolFingerprint( + protocol: FixedTraceEvaluationProtocol, +): string { + return sha256(validatedFixedTraceEvaluationProtocol(protocol)); +} +export function assertFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, +): void { + void validatedFixedTraceEvaluationProtocol(protocol); +} +function validatedFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, +): FixedTraceEvaluationProtocol { + const snapshot = snapshotFixedTraceJson( + protocol, + "fixed-trace evaluation protocol", + ) as FixedTraceEvaluationProtocol; + validateFixedTraceEvaluationProtocol(snapshot); + return snapshot; +} +function validateFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, +): void { + assertFixedTracePartitionManifest(); + if ( + FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "router").length !== + FIXED_TRACE_ARCHITECTURE_CELL_TRUTH.routerCells || + FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "generation") + .length !== FIXED_TRACE_ARCHITECTURE_CELL_TRUTH.generationCells + ) + throw new Error("executable router/generator cell inventory differs from pinned planning truth"); + if ( + protocol.version !== FIXED_TRACE_EVALUATION_PROTOCOL_VERSION || + protocol.baseCapabilityUniverse !== + "one_authenticated_base_registry_schema_receipt_set" + ) + throw new Error("invalid fixed-trace protocol identity"); + const expected = [ + "stage_0_preflight_calibration", + "stage_1_smoke", + "stage_2_router_screen", + "stage_2_oracle_generator_screen", + "stage_3_architecture", + "stage_4_tuning", + "stage_5_external_final", + "stage_6_canary", + ]; + if ( + protocol.phases.length !== expected.length || + protocol.phases.some((phase, index) => phase.id !== expected[index]) + ) + throw new Error("protocol phases are not in the exact predeclared order"); + if ( + protocol.finalProtocol.status !== "unavailable" || + protocol.finalProtocol.externalN !== null || + protocol.finalProtocol.externalPackDigest !== null || + protocol.finalProtocol.candidatePipelineId !== null || + protocol.finalProtocol.comparatorPipelineId !== null || + protocol.finalProtocol.architectureArmId !== null || + protocol.finalProtocol.fingerprint !== null || + protocol.finalProtocol.powerResult !== null + ) + throw new Error( + "external final is unavailable until exact paired-discordance power is fingerprinted", + ); + if ( + FIXED_TRACE_CONFIRMATORY_ADMISSION.status !== + "not_admitted_missing_fingerprinted_statistical_protocol" || + FIXED_TRACE_CONFIRMATORY_ADMISSION.finalProtocolFingerprint !== null || + FIXED_TRACE_CONFIRMATORY_ADMISSION.sizingPilot.digest !== null || + FIXED_TRACE_CONFIRMATORY_ADMISSION.judgeCalibration.digest !== null + ) { + throw new Error( + "confirmatory statistical admission must fail closed until independently custodied", + ); + } + for (const phase of protocol.phases) { + const expectedCases = + phase.caseSet === "development" + ? FIXED_TRACE_PARTITION_MANIFEST.development.length + : phase.caseSet === "tuning" + ? FIXED_TRACE_PARTITION_MANIFEST.tuning.length + : null; + if ( + phase.uniqueCases !== null && + phase.uniqueCases !== 8 && + phase.uniqueCases !== expectedCases + ) + throw new Error( + `phase ${phase.id} does not use corpus-derived case counts`, + ); + if (phase.id === "stage_1_smoke") { + if ( + phase.uniqueCases !== FIXED_TRACE_COMPONENT_SMOKE_PLAN.cases || + phase.repetitions !== 1 || + phase.arms.length !== FIXED_TRACE_COMPONENT_SMOKE_PLAN.totalComponentCells || + phase.arms.some( + (arm) => + arm.architecture !== "none" || + arm.stages.length !== 1 || + arm.stages[0]!.role === "judge" || + (arm.stages[0]!.role === "generation" && + arm.stages[0]!.maxInvocationsPerCase !== + FIXED_TRACE_COMPONENT_SMOKE_PLAN.maxGenerationInvocationsPerCase), + ) + ) + throw new Error("stage_1 is only the pinned component-only smoke"); + } + for (const arm of phase.arms) { + if ( + phase.selectionUse === "architecture_selection" && + arm.architecture === "two_stage_llm_router" && + arm.admission !== "not_admitted_common_tool_universe" + ) + throw new Error( + "architecture comparison remains not admitted without a common authenticated tool universe", + ); + if ( + arm.architecture === "direct_generation" && + arm.admission !== "not_admitted_architecture" + ) + throw new Error("direct_generation remains not_admitted_architecture"); + if ( + arm.architecture === "deterministic_policy_llm_fallback_hybrid" && + (!arm.conditionalCalls || + !arm.stages.some((item) => item.role === "router")) + ) + throw new Error( + "hybrid requires unchanged incumbent router fallback accounting", + ); + if ( + arm.architecture === "deterministic_policy_llm_fallback_hybrid" && + !FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT.find( + (preflight) => preflight.phase === "development", + )!.evaluable && + arm.admission !== "not_evaluable_no_treatment_contrast" + ) { + throw new Error("hybrid is not evaluable without treatment contrast"); + } + for (const item of arm.stages) + if ( + item.cellId !== null && + !FIXED_TRACE_ADMITTED_CELLS.some((cell) => cell.id === item.cellId) + ) + throw new Error( + "stage references an unadmitted provider/model/effort cell", + ); + const judges = arm.stages.filter((item) => item.role === "judge"); + if (judges.length && arm.admission === "admitted_diagnostic") { + const expectedJudges = assertPromotionGradeDualJudgeFeasibility( + arm, + ).map((cell) => cell.id); + if ( + judges.length !== 2 || + judges + .map((item) => item.cellId) + .some((id, index) => id !== expectedJudges[index]) + ) + throw new Error( + "semantic judges must be the two calibrated providers excluding every pipeline provider", + ); + } + } + } + // This exported plan is a pinned declaration, not a caller-editable schema. + // Validate every nested field before it can be fingerprinted or budgeted. + if (sha256(protocol) !== sha256(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL)) + throw new Error("fixed-trace protocol differs from the pinned declaration"); +} +export function estimateFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, +): FixedTraceProtocolEstimate { + protocol = validatedFixedTraceEvaluationProtocol(protocol); + const stages: FixedTraceStageCeiling[] = []; + for (const phase of protocol.phases) + for (const arm of phase.arms) { + if ( + (arm.admission === "not_admitted_architecture" || + phase.uniqueCases === null) && + arm.architecture !== "deterministic_policy_llm_fallback_hybrid" + ) + continue; + const uniqueCases = phase.uniqueCases; + if (uniqueCases === null) continue; + for (const item of arm.stages) { + const cell = FIXED_TRACE_ADMITTED_CELLS.find( + (entry) => entry.id === item.cellId, + ); + if (!cell) continue; + const profile = FIXED_TRACE_PROTOCOL_PRICING.find( + (entry) => entry.profileId === cell.pricingProfileId, + )!; + validateFixedTracePricing(profile); + const calls = + uniqueCases * phase.repetitions * item.maxInvocationsPerCase; + stages.push({ + phaseId: phase.id, + armId: arm.id, + role: item.role, + calls, + ceilingUsd: fixedTraceEstimatedCostUsd( + { + inputTokens: calls * item.maxInputTokensPerInvocation, + outputTokens: calls * item.maxOutputTokensPerInvocation, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + profile, + ), + }); + } + } + const candidateCeilingUsd = stages + .filter((item) => item.role === "router" || item.role === "generation") + .reduce((total, item) => total + item.ceilingUsd, 0); + const judgeCeilingUsd = stages + .filter((item) => item.role === "judge") + .reduce((total, item) => total + item.ceilingUsd, 0); + const componentSmokeCeilingUsd = stages + .filter((item) => item.phaseId === "stage_1_smoke") + .reduce((total, item) => total + item.ceilingUsd, 0); + if (componentSmokeCeilingUsd > FIXED_TRACE_COMPONENT_SMOKE_PLAN.providerCeilingUsd) + throw new Error("component-only smoke exceeds its non-authorizing $5 provider ceiling"); + const failedTimeoutUnknownExposureCeilingUsd = + candidateCeilingUsd + judgeCeilingUsd; + const contingencyUsd = (candidateCeilingUsd + judgeCeilingUsd) * 0.1; + const hybridRouter = stages.find( + (item) => + item.phaseId === "stage_3_architecture" && + item.armId === "hybrid-locked-finalist" && + item.role === "router", + )!; + const architecturePhase = protocol.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + const developmentContrast = FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT.find( + (preflight) => preflight.phase === "development", + )!; + const armCallAccounting = architecturePhase.arms.map((arm) => { + const router = arm.stages.find((item) => item.role === "router"); + const generation = arm.stages.find((item) => item.role === "generation"); + const localTerminalCases = + arm.architecture === "deterministic_policy_llm_fallback_hybrid" + ? developmentContrast.localTerminalCases * architecturePhase.repetitions + : 0; + const routedCases = + arm.architecture === "direct_generation" + ? 0 + : architecturePhase.uniqueCases! * architecturePhase.repetitions - + localTerminalCases; + const cost = (item: FixedTraceProtocolStage | undefined, calls: number) => { + if (!item?.cellId) return 0; + const cell = FIXED_TRACE_ADMITTED_CELLS.find( + (entry) => entry.id === item.cellId, + )!; + const profile = FIXED_TRACE_PROTOCOL_PRICING.find( + (entry) => entry.profileId === cell.pricingProfileId, + )!; + return fixedTraceEstimatedCostUsd( + { + inputTokens: calls * item.maxInputTokensPerInvocation, + outputTokens: calls * item.maxOutputTokensPerInvocation, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + profile, + ); + }; + const routerCalls = router ? routedCases * router.maxInvocationsPerCase : 0; + const generationCalls = generation + ? routedCases * generation.maxInvocationsPerCase + : 0; + return Object.freeze({ + armId: arm.id, + admission: arm.admission, + evaluable: + arm.admission === "admitted_diagnostic" && + (arm.architecture !== "deterministic_policy_llm_fallback_hybrid" || + developmentContrast.evaluable), + localTerminalCases, + routedCases, + routerCalls, + generationCalls, + routerCeilingUsd: cost(router, routerCalls), + generationCeilingUsd: cost(generation, generationCalls), + }); + }); + return Object.freeze({ + dispatchable: false, + approvalCeilingUsd: null, + stages: Object.freeze(stages), + candidateCeilingUsd, + judgeCeilingUsd, + simulatorCeilingUsd: 0, + failedTimeoutUnknownExposureCeilingUsd, + contingencyUsd, + totalCeilingUsd: candidateCeilingUsd + judgeCeilingUsd + contingencyUsd, + componentSmokeCeilingUsd, + hybridWorstCaseRouterCalls: 138, + hybridWorstCaseRouterCeilingUsd: hybridRouter.ceilingUsd, + armCallAccounting: Object.freeze(armCallAccounting), + externalFinalN: null, + }); +} +/** Promotion-grade semantic inference excludes every LLM used in the pipeline. */ +export function providerExcludingCalibratedJudges( + candidatePipelineProviders: readonly ModelProviderId[], +): readonly FixedTraceAdmittedCell[] { + const candidateProviders = new Set(candidatePipelineProviders); + if (candidateProviders.size !== 1) { + throw new Error( + "promotion-grade dual-LLM judging requires a single-provider complete pipeline; mixed finalists require a human-primary path or fourth calibrated provider", + ); + } + const selected = + (["anthropic", "openai", "google"] as const) + .filter((provider) => !candidateProviders.has(provider)) + .map((provider) => + FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => + cell.role === "generation" && + cell.provider === provider && + (provider === "openai" + ? cell.effort === "none" + : cell.effort === "provider_default"), + ), + ); + if ( + selected.length !== 2 || + selected.some((cell) => !cell) || + selected.some((cell) => { + const calibration = FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS.find( + (entry) => + entry.provider === cell!.provider && + entry.model === cell!.model && + entry.effort === cell!.effort, + ); + return !calibration || calibration.authenticatedAdmission === null; + }) + ) + throw new Error( + "no admissible provider-excluding judge pair without calibrated custodied artifacts", + ); + return Object.freeze(selected as FixedTraceAdmittedCell[]); +} +export function semanticJudgeCandidateProviders( + arm: Pick, +): readonly ModelProviderId[] { + const providers = arm.stages + .filter((stage) => stage.role === "router" || stage.role === "generation") + .map((stage) => + FIXED_TRACE_ADMITTED_CELLS.find((cell) => cell.id === stage.cellId), + ); + if ( + !providers.some((cell) => cell && cell.role === "generation") || + providers.some((cell) => !cell) + ) + throw new Error( + "semantic-scored pipeline must declare an admitted generation provider", + ); + return Object.freeze([...new Set(providers.map((cell) => cell!.provider))]); +} +export function assertPromotionGradeDualJudgeFeasibility( + arm: Pick, +): readonly FixedTraceAdmittedCell[] { + return providerExcludingCalibratedJudges( + semanticJudgeCandidateProviders(arm), + ); +} diff --git a/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts new file mode 100644 index 0000000000..6a37fe8de1 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-evaluator-coordinator.ts @@ -0,0 +1,487 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { types } from "node:util"; +import type { + ModelProviderId, + ModelReasoningEffort, +} from "../model-providers/model-provider.js"; +import { snapshotFixedTraceJson } from "./fixed-trace-safe-snapshot.js"; + +/** + * Diagnostic integrity only. An importer supplies this module's key, so this + * cannot establish evaluator custody or confirmatory evidence. A separately + * injected opaque privileged signer and durable dispatcher/ledger boundary is + * required before any admission can be made. + */ +export const FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION = + "addie-fixed-trace-evaluator-coordinator-v1" as const; + +export type FixedTraceLedgerTamperClass = + | "omission" + | "insertion" + | "duplication" + | "substitution" + | "reordering" + | "authentication" + | "unknown_exposure"; +export class FixedTraceLedgerValidationError extends Error { + constructor( + readonly tamperClass: FixedTraceLedgerTamperClass, + message: string, + ) { + super(message); + this.name = "FixedTraceLedgerValidationError"; + } +} + +export interface FixedTraceExpectedInvocation { + readonly runId: string; + readonly phaseId: string; + readonly caseId: string; + readonly armId: string; + readonly stage: "router" | "generation" | "judge" | "simulator"; + readonly invocation: number; + readonly attempt: number; + readonly requested: { + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly identityPolicy: string; + }; + readonly controls: { + readonly promptSha256: string; + readonly systemSha256: string; + readonly messagesSha256: string; + readonly toolSchemaSha256: string; + readonly providerRequestSha256: string; + readonly presentedToolNames: readonly string[]; + readonly presentedToolOrderSha256: string; + readonly simulatorReceiptProvenanceSha256: string; + readonly simulatorControlsSha256: string; + readonly architectureSha256: string; + readonly admissionSha256: string; + readonly configSha256: string; + readonly pricingSha256: string; + readonly limitsSha256: string; + readonly retryCacheSamplingSha256: string; + readonly failureDenominatorId: string; + }; +} +export interface FixedTraceActualInvocation extends FixedTraceExpectedInvocation { + readonly returned: { + readonly provider: ModelProviderId | null; + readonly model: string | null; + readonly identityPolicy: string | null; + }; + readonly toolCallsSha256: string | null; + readonly toolInputsSha256: string | null; + readonly toolResultsSha256: string | null; + readonly startedAt: string; + readonly finishedAt: string; + readonly latencyMs: number; + readonly usage: { + readonly inputTokens: number; + readonly outputTokens: number; + readonly cacheReadTokens: number; + readonly cacheWriteTokens: number; + } | null; + readonly pricing: { + readonly profileId: string; + readonly costUsd: number; + } | null; + readonly terminalStatus: + | "complete" + | "timeout_after_dispatch" + | "provider_error" + | "malformed" + | "empty" + | "truncated" + | "tool_boundary" + | "privacy_violation" + | "not_dispatched_budget" + | "unknown_exposure"; + readonly errorCode: string | null; +} +export interface FixedTraceExpectedSequenceContract { + readonly version: typeof FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION; + readonly keyId: string; + readonly runId: string; + readonly protocolFingerprint: string; + readonly manifestFingerprint: string; + readonly entries: readonly FixedTraceExpectedInvocation[]; + readonly signature: string; +} +export interface FixedTraceEvidenceLedger { + readonly admission: "not_admitted_diagnostic_hmac_without_privileged_durable_authority"; + readonly contract: FixedTraceExpectedSequenceContract; + readonly entries: readonly FixedTraceActualInvocation[]; + readonly complete: boolean; + readonly halted: boolean; + readonly plannedDenominator: number; + readonly observedDenominator: number; + readonly hardFailureDenominator: number; + readonly signature: string; +} + +const DIAGNOSTIC_ADMISSION = + "not_admitted_diagnostic_hmac_without_privileged_durable_authority" as const; +const hasExactKeys = (value: unknown, keys: readonly string[]) => + typeof value === "object" && + value !== null && + Object.keys(value).sort().join(",") === [...keys].sort().join(","); + +function canonical(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new Error("non-finite evaluator ledger value"); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`) + .join(",")}}`; + } + throw new Error("non-JSON evaluator ledger value"); +} +function deepSnapshot(value: T): T { + return snapshotFixedTraceJson(value, "fixed-trace evaluator coordinator") as T; +} +function snapshotCoordinatorConfig(value: unknown): { + readonly hmacKey: Uint8Array; + readonly keyId: string; +} { + if ( + typeof value !== "object" || + value === null || + types.isProxy(value) || + Object.getPrototypeOf(value) !== Object.prototype || + Object.getOwnPropertySymbols(value).length !== 0 + ) + throw new Error("evaluator coordinator configuration must be a plain non-proxy object"); + const descriptors = Object.getOwnPropertyDescriptors(value); + if (Object.keys(descriptors).sort().join(",") !== "hmacKey,keyId") + throw new Error("evaluator coordinator configuration has extra or missing fields"); + const key = descriptors.hmacKey; + const keyId = descriptors.keyId; + if (!key || !("value" in key) || !keyId || !("value" in keyId)) + throw new Error("evaluator coordinator configuration must use data properties"); + if ( + types.isProxy(key.value) || + !(key.value instanceof Uint8Array) || + key.value.byteLength < 32 || + typeof keyId.value !== "string" || + !keyId.value.trim() + ) + throw new Error("evaluator-owned HMAC custody configuration is required"); + return Object.freeze({ hmacKey: new Uint8Array(key.value), keyId: keyId.value }); +} +const isProvider = (value: unknown): value is ModelProviderId => + value === "anthropic" || value === "openai" || value === "google"; +const isEffort = (value: unknown): value is ModelReasoningEffort => + value === "provider_default" || value === "none" || value === "low" || value === "medium" || value === "high"; +const isNonemptyString = (value: unknown): value is string => + typeof value === "string" && value.trim().length > 0; +function assertExpectedInvocation(entry: FixedTraceExpectedInvocation, runId: string): void { + if (!hasExactKeys(entry, [ + "runId", "phaseId", "caseId", "armId", "stage", "invocation", "attempt", "requested", "controls", + ])) throw new Error("expected sequence entry has extra or missing fields"); + if ( + entry.runId !== runId || + !isNonemptyString(entry.phaseId) || + !isNonemptyString(entry.caseId) || + !isNonemptyString(entry.armId) || + !["router", "generation", "judge", "simulator"].includes(entry.stage) || + !Number.isSafeInteger(entry.invocation) || entry.invocation < 1 || + !Number.isSafeInteger(entry.attempt) || entry.attempt < 1 + ) throw new Error("expected sequence entry has invalid cross-field identity"); + if (!hasExactKeys(entry.requested, ["provider", "model", "effort", "identityPolicy"]) || + !isProvider(entry.requested.provider) || !isNonemptyString(entry.requested.model) || + !isEffort(entry.requested.effort) || !isNonemptyString(entry.requested.identityPolicy)) + throw new Error("expected sequence entry has invalid requested identity"); + const controls = entry.controls; + if (!hasExactKeys(controls, [ + "promptSha256", "systemSha256", "messagesSha256", "toolSchemaSha256", "providerRequestSha256", + "presentedToolNames", "presentedToolOrderSha256", "simulatorReceiptProvenanceSha256", + "simulatorControlsSha256", "architectureSha256", "admissionSha256", "configSha256", "pricingSha256", + "limitsSha256", "retryCacheSamplingSha256", "failureDenominatorId", + ]) || + !Object.entries(controls).every(([key, value]) => key === "presentedToolNames" + ? Array.isArray(value) && value.every(isNonemptyString) + : isNonemptyString(value))) + throw new Error("expected sequence entry has invalid controls"); +} +const invocationKey = ( + entry: Pick< + FixedTraceExpectedInvocation, + | "runId" + | "phaseId" + | "caseId" + | "armId" + | "stage" + | "invocation" + | "attempt" + >, +) => + [ + entry.runId, + entry.phaseId, + entry.caseId, + entry.armId, + entry.stage, + entry.invocation, + entry.attempt, + ].join("\u0000"); +const contractProjection = ( + contract: Omit, +) => canonical(contract); +const ledgerProjection = (ledger: Omit) => + canonical(ledger); +const sameExpected = ( + actual: FixedTraceActualInvocation, + expected: FixedTraceExpectedInvocation, +) => { + const { + returned, + toolCallsSha256, + toolInputsSha256, + toolResultsSha256, + startedAt, + finishedAt, + latencyMs, + usage, + pricing, + terminalStatus, + errorCode, + ...requested + } = actual; + void returned; + void toolCallsSha256; + void toolInputsSha256; + void toolResultsSha256; + void startedAt; + void finishedAt; + void latencyMs; + void usage; + void pricing; + void terminalStatus; + void errorCode; + return canonical(requested) === canonical(expected); +}; + +export function createFixedTraceEvaluatorCoordinator(evaluatorConfig: { + readonly hmacKey: Uint8Array; + readonly keyId: string; +}) { + const detachedConfig = snapshotCoordinatorConfig(evaluatorConfig); + const sign = (projection: string) => + createHmac("sha256", detachedConfig.hmacKey) + .update( + `${FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION}\u0000${detachedConfig.keyId}\u0000${projection}`, + ) + .digest("hex"); + const verify = (contract: FixedTraceExpectedSequenceContract) => { + if (!hasExactKeys(contract, [ + "version", "keyId", "runId", "protocolFingerprint", "manifestFingerprint", "entries", "signature", + ])) + throw new FixedTraceLedgerValidationError( + "authentication", + "expected sequence contract has extra or missing fields", + ); + const projection = contractProjection({ + version: contract.version, + keyId: contract.keyId, + runId: contract.runId, + protocolFingerprint: contract.protocolFingerprint, + manifestFingerprint: contract.manifestFingerprint, + entries: contract.entries, + }); + const expected = Buffer.from(sign(projection), "hex"); + const supplied = Buffer.from(contract.signature, "hex"); + if ( + contract.version !== FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION || + contract.keyId !== detachedConfig.keyId || + expected.length !== supplied.length || + !timingSafeEqual(expected, supplied) + ) + throw new FixedTraceLedgerValidationError( + "authentication", + "expected sequence contract authentication failed", + ); + }; + return Object.freeze({ + admission: DIAGNOSTIC_ADMISSION, + issueExpectedSequence( + input: Omit< + FixedTraceExpectedSequenceContract, + "version" | "keyId" | "signature" + >, + ): FixedTraceExpectedSequenceContract { + input = deepSnapshot(input); + if (!hasExactKeys(input, ["runId", "protocolFingerprint", "manifestFingerprint", "entries"])) + throw new Error("expected sequence has extra or missing fields"); + if ( + !input.runId.trim() || + !input.protocolFingerprint.trim() || + !input.manifestFingerprint.trim() || + input.entries.length === 0 + ) + throw new Error( + "complete evaluator-owned expected sequence is required before dispatch", + ); + const keys = new Set(); + for (const entry of input.entries) { + assertExpectedInvocation(entry, input.runId); + const key = invocationKey(entry); + if (entry.runId !== input.runId || keys.has(key)) + throw new Error( + "expected sequence has a duplicate or wrong-run invocation", + ); + keys.add(key); + } + const unsigned = deepSnapshot({ + version: FIXED_TRACE_EVALUATOR_COORDINATOR_VERSION, + keyId: detachedConfig.keyId, + ...input, + entries: deepSnapshot(input.entries), + } as const); + return deepSnapshot({ + ...unsigned, + signature: sign(contractProjection(unsigned)), + }); + }, + validate( + contract: FixedTraceExpectedSequenceContract, + actualEntries: readonly FixedTraceActualInvocation[], + ): FixedTraceEvidenceLedger { + const trustedContract = deepSnapshot(contract); + const trustedActualEntries = deepSnapshot(actualEntries); + verify(trustedContract); + const observed: FixedTraceActualInvocation[] = []; + const seen = new Set(); + let halted = false; + for (const actual of trustedActualEntries) { + if (halted) + throw new FixedTraceLedgerValidationError( + "unknown_exposure", + "run was halted after unknown exposure", + ); + const key = invocationKey(actual); + const expected = trustedContract.entries[observed.length]; + const knownIndex = trustedContract.entries.findIndex( + (entry) => invocationKey(entry) === key, + ); + if (seen.has(key)) + throw new FixedTraceLedgerValidationError( + "duplication", + "ledger duplicated an invocation", + ); + if (knownIndex < 0) + throw new FixedTraceLedgerValidationError( + "insertion", + "ledger inserted an unplanned invocation", + ); + if (!expected) + throw new FixedTraceLedgerValidationError( + "insertion", + "ledger exceeded the pre-dispatch expected sequence", + ); + if (knownIndex > observed.length) { + const expectedKey = invocationKey(expected); + const appearsLater = trustedActualEntries + .slice(observed.length + 1) + .some((entry) => invocationKey(entry) === expectedKey); + throw new FixedTraceLedgerValidationError( + appearsLater ? "reordering" : "omission", + appearsLater + ? "ledger reordered an expected invocation" + : "ledger omitted an expected invocation before a later one", + ); + } + if (knownIndex < observed.length) + throw new FixedTraceLedgerValidationError( + "reordering", + "ledger reordered an expected invocation", + ); + if (!sameExpected(actual, expected)) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger substituted evaluator-owned requested identity or controls", + ); + if ( + !Number.isFinite(actual.latencyMs) || + actual.latencyMs < 0 || + Number.isNaN(Date.parse(actual.startedAt)) || + Number.isNaN(Date.parse(actual.finishedAt)) + ) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger has invalid timing evidence", + ); + if (Date.parse(actual.finishedAt) < Date.parse(actual.startedAt)) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger finished before it started", + ); + if (actual.terminalStatus === "unknown_exposure") { + halted = true; + throw new FixedTraceLedgerValidationError( + "unknown_exposure", + "unknown provider exposure halts the run", + ); + } + if (actual.terminalStatus !== "not_dispatched_budget") { + if ( + actual.returned.provider !== expected.requested.provider || + actual.returned.model !== expected.requested.model || + actual.returned.identityPolicy !== expected.requested.identityPolicy + ) + throw new FixedTraceLedgerValidationError( + "substitution", + "ledger returned a different provider, model, or identity policy", + ); + if ( + !actual.usage || + !actual.pricing || + !Number.isFinite(actual.pricing.costUsd) || + actual.pricing.costUsd < 0 || + Object.values(actual.usage).some( + (value) => !Number.isSafeInteger(value) || value < 0, + ) + ) + throw new FixedTraceLedgerValidationError( + "substitution", + "dispatched invocation lacks complete trusted usage or pricing", + ); + } + seen.add(key); + observed.push(actual); + } + if (observed.length !== trustedContract.entries.length) + throw new FixedTraceLedgerValidationError( + "omission", + "ledger ended before its planned denominator", + ); + const hardFailureDenominator = observed.filter( + (entry) => entry.terminalStatus !== "complete", + ).length; + const unsignedLedger = deepSnapshot({ + admission: DIAGNOSTIC_ADMISSION, + contract: trustedContract, + entries: observed, + complete: true, + halted: false, + plannedDenominator: trustedContract.entries.length, + observedDenominator: observed.length, + hardFailureDenominator, + }); + return deepSnapshot({ + ...unsignedLedger, + signature: sign(ledgerProjection(unsignedLedger)), + }); + }, + }); +} diff --git a/server/src/addie/eval/fixed-trace-judge.ts b/server/src/addie/eval/fixed-trace-judge.ts index 294379ca54..73fa3cabff 100644 --- a/server/src/addie/eval/fixed-trace-judge.ts +++ b/server/src/addie/eval/fixed-trace-judge.ts @@ -22,9 +22,20 @@ import type { export const FIXED_TRACE_JUDGE_PROMPT_VERSION = 'addie-fixed-trace-blinded-judge-v2'; export const FIXED_TRACE_MIN_INDEPENDENT_JUDGES = 2; +/** + * There is no privileged calibration custody boundary in this integration + * draft. A caller-provided hash or boolean cannot satisfy this admission. + */ +export const FIXED_TRACE_JUDGE_CALIBRATION_ADMISSION = + 'not_admitted_missing_privileged_custodied_calibration' as const; +function hasPrivilegedCustodiedCalibration(): boolean { + return false; +} const MAX_JUDGE_INPUT_BYTES = 24 * 1024; const MAX_JUDGE_OUTPUT_BYTES = 8 * 1024; +const isModelProviderId = (value: unknown): value is ModelProviderId => + value === 'anthropic' || value === 'openai' || value === 'google'; const FIXED_TRACE_JUDGE_VERDICT_SCHEMA: Readonly = Object.freeze({ type: 'object', properties: { @@ -65,6 +76,7 @@ export type FixedTraceJudgeStatus = export type FixedTraceJudgeFailureReason = | 'candidate_not_judgeable' | 'judge_not_independent' + | 'judge_calibration_not_admitted' | 'judge_input_out_of_bounds' | 'judge_output_truncated' | 'judge_output_invalid' @@ -331,18 +343,47 @@ function validateConfig(config: FixedTraceJudgeConfig): void { ) throw new Error('Judge pricing is invalid'); } -function candidateProviders(observation: FixedTraceObservation): ReadonlySet { - const generation = [ - observation.metadata.generation.requestedProvider, - observation.metadata.generation.returnedProvider, - ].filter((provider): provider is ModelProviderId => provider !== null); - const providers = generation.length > 0 - ? generation - : [ - observation.metadata.router.requestedProvider, - observation.metadata.router.returnedProvider, - ].filter((provider): provider is ModelProviderId => provider !== null); - return new Set(providers); +function candidateProviders( + observation: FixedTraceObservation, +): ReadonlySet | null { + const stages = [observation.metadata.router, observation.metadata.generation]; + const providers = new Set(); + for (const stage of stages) { + if (!stage.providerExposures) return null; + if (stage.source === 'provider' && stage.providerExposures.length === 0) return null; + if (stage.providerExposures.length !== stage.dispatchedCalls) return null; + const attempts = new Set(); + const preparedIdentities = new Set(); + const returnedIdentities = new Set(); + for (const exposure of stage.providerExposures) { + if ( + !Number.isSafeInteger(exposure.attempt) || + exposure.attempt < 1 || + !exposure.preparedModel || + exposure.attempt > stage.dispatchedCalls || + !isModelProviderId(exposure.preparedProvider) || + (exposure.returnedProvider === null) !== (exposure.returnedModel === null) || + (exposure.returnedProvider !== null && !isModelProviderId(exposure.returnedProvider)) || + (exposure.returnedModel !== null && !exposure.returnedModel) + ) return null; + const preparedIdentity = `${exposure.preparedProvider}\u0000${exposure.preparedModel}`; + if (attempts.has(exposure.attempt)) return null; + attempts.add(exposure.attempt); + preparedIdentities.add(preparedIdentity); + providers.add(exposure.preparedProvider); + if (exposure.returnedProvider) { + returnedIdentities.add(`${exposure.returnedProvider}\u0000${exposure.returnedModel}`); + providers.add(exposure.returnedProvider); + } + } + if ( + (stage.requestedProvider === null) !== (stage.requestedModel === null) || + (stage.returnedProvider === null) !== (stage.returnedModel === null) || + (stage.requestedProvider !== null && !preparedIdentities.has(`${stage.requestedProvider}\u0000${stage.requestedModel}`)) || + (stage.returnedProvider !== null && !returnedIdentities.has(`${stage.returnedProvider}\u0000${stage.returnedModel}`)) + ) return null; + } + return providers.size ? providers : null; } export async function judgeFixedTraceObservation( @@ -376,7 +417,7 @@ export async function judgeFixedTraceObservation( if ( !trace.answerRubric?.length || observation.terminalStatus !== 'complete' - || candidateProviderIds.size === 0 + || candidateProviderIds === null ) { return { traceId: trace.id, @@ -395,6 +436,18 @@ export async function judgeFixedTraceObservation( metadata: metadata(config, request, [], false, startedAt, null), }; } + // Do not let a planning-side calibration record authorize a provider call. + // A separately injected privileged, custodied calibration verifier is the + // prerequisite; this module intentionally has no caller-mintable seam. + if (!hasPrivilegedCustodiedCalibration()) { + return { + traceId: trace.id, + status: 'skipped', + failureReason: 'judge_calibration_not_admitted', + verdict: null, + metadata: metadata(config, request, [], false, startedAt, null), + }; + } const invocations: PreparedModelInvocation[] = []; let dispatched = false; @@ -460,6 +513,8 @@ export async function runIndependentFixedTraceJudges( observations: ReadonlyArray, judgeConfigs: ReadonlyArray, ): Promise { + if (!hasPrivilegedCustodiedCalibration()) + throw new Error('independent judge dispatch is not admitted without privileged custodied calibration'); const configsByProvider = new Map(); for (const config of judgeConfigs) { if (configsByProvider.has(config.provider.id)) throw new Error('Independent judges must use unique providers'); @@ -471,6 +526,8 @@ export async function runIndependentFixedTraceJudges( const observation = observationsById.get(trace.id); if (!observation) continue; const candidateProviderIds = candidateProviders(observation); + if (!candidateProviderIds) + throw new Error(`Trace ${trace.id} has incomplete candidate provider exposure`); const independentConfigs = judgeConfigs.filter((config) => !candidateProviderIds.has(config.provider.id)); if (independentConfigs.length < FIXED_TRACE_MIN_INDEPENDENT_JUDGES) { throw new Error(`Trace ${trace.id} requires at least two independent judge providers`); @@ -506,9 +563,11 @@ export function summarizeFixedTraceJudges( for (const trace of applicable) { const group = byTrace.get(trace.id) ?? []; const providers = new Set(group.map((judgment) => judgment.metadata.requestedProvider)); - const candidates = candidateProviderIds.get(trace.id) ?? new Set(); + const candidates = candidateProviderIds.get(trace.id); const complete = group.length >= FIXED_TRACE_MIN_INDEPENDENT_JUDGES && providers.size === group.length + && candidates !== null + && candidates !== undefined && candidates.size > 0 && [...providers].every((provider) => !candidates.has(provider)) && group.every((judgment) => judgment.status === 'judged' && judgment.verdict !== null); @@ -529,6 +588,7 @@ export function summarizeFixedTraceJudges( const ratio = (count: number, denominator: number) => denominator === 0 ? 0 : count / denominator; const judgedJudgments = judgments.filter((judgment) => judgment.status === 'judged').length; const comparisonEligible = applicable.length > 0 + && hasPrivilegedCustodiedCalibration() && completeCases.every(Boolean) && judgments.length === expectedJudgments && totalEstimatedCostUsd !== null; diff --git a/server/src/addie/eval/fixed-trace-partition.ts b/server/src/addie/eval/fixed-trace-partition.ts new file mode 100644 index 0000000000..e98f7a8b4a --- /dev/null +++ b/server/src/addie/eval/fixed-trace-partition.ts @@ -0,0 +1,76 @@ +import { createHash } from "node:crypto"; +import { + FIXED_TRACE_CORPUS, + FIXED_TRACE_PHASE_COUNTS, +} from "./fixed-trace-suite.js"; + +/** + * This ID-only manifest is the partition boundary. It deliberately contains + * no fixture text, expected routes, or grading rubric. + */ +export const FIXED_TRACE_PARTITION_MANIFEST_VERSION = + "addie-fixed-trace-partition-v2" as const; +export const FIXED_TRACE_PARTITION_MANIFEST = Object.freeze({ + version: FIXED_TRACE_PARTITION_MANIFEST_VERSION, + /** Derived from the corpus authority, never a stale hand-copied list. */ + development: Object.freeze( + FIXED_TRACE_CORPUS.filter((trace) => trace.phase === "development").map( + (trace) => trace.id, + ), + ), + tuning: Object.freeze( + FIXED_TRACE_CORPUS.filter((trace) => trace.phase === "tuning").map( + (trace) => trace.id, + ), + ), +}); + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + throw new Error("Partition manifest contains a non-JSON value"); +} + +export function fixedTracePartitionManifestSha256(): string { + return createHash("sha256") + .update(canonicalJson(FIXED_TRACE_PARTITION_MANIFEST), "utf8") + .digest("hex"); +} + +export const FIXED_TRACE_PARTITION_MANIFEST_SHA256 = + fixedTracePartitionManifestSha256(); + +export function assertFixedTracePartitionManifest(): void { + if ( + fixedTracePartitionManifestSha256() !== + FIXED_TRACE_PARTITION_MANIFEST_SHA256 + ) { + throw new Error("Fixed-trace partition manifest hash mismatch"); + } + const all = [ + ...FIXED_TRACE_PARTITION_MANIFEST.development, + ...FIXED_TRACE_PARTITION_MANIFEST.tuning, + ]; + if (new Set(all).size !== all.length) + throw new Error("Fixed-trace partition manifest has duplicate IDs"); + if ( + FIXED_TRACE_PARTITION_MANIFEST.development.length !== + FIXED_TRACE_PHASE_COUNTS.development || + FIXED_TRACE_PARTITION_MANIFEST.tuning.length !== + FIXED_TRACE_PHASE_COUNTS.tuning || + FIXED_TRACE_PHASE_COUNTS.sealed_final !== 0 || + all.length !== 82 + ) { + throw new Error( + "Fixed-trace partition manifest does not match the 46 development / 36 tuning corpus authority", + ); + } +} diff --git a/server/src/addie/eval/fixed-trace-runner.ts b/server/src/addie/eval/fixed-trace-runner.ts index 1cbbdf5839..f42b6c56a8 100644 --- a/server/src/addie/eval/fixed-trace-runner.ts +++ b/server/src/addie/eval/fixed-trace-runner.ts @@ -332,6 +332,25 @@ interface StageInvocationState { latencyMs: number; } +function providerExposures( + state: StageInvocationState, + response?: ModelResponse, + recordedExposures?: NonNullable, +): FixedTraceModelStageMetadata["providerExposures"] { + if (recordedExposures) return deepFreeze(recordedExposures.map((exposure) => ({ ...exposure }))); + return deepFreeze( + state.invocations.map((prepared, index) => ({ + attempt: index + 1, + preparedProvider: prepared.provider, + preparedModel: prepared.model, + returnedProvider: + response && index === state.invocations.length - 1 ? response.provider : null, + returnedModel: + response && index === state.invocations.length - 1 ? response.model : null, + })), + ); +} + function canonicalJson(value: unknown): string { if (value === null || typeof value === 'boolean' || typeof value === 'string') return JSON.stringify(value); if (typeof value === 'number') { @@ -459,6 +478,7 @@ function providerStageMetadata( response: ModelResponse, usage: ModelUsage, state: StageInvocationState, + recordedExposures?: NonNullable, ): FixedTraceModelStageMetadata { // Provider responses are outside evaluator ownership. Retaining their usage // object would let a later provider turn mutate already-recorded cost and @@ -473,6 +493,7 @@ function providerStageMetadata( requestedModel: config.model, returnedProvider: response.provider, returnedModel: response.model, + providerExposures: providerExposures(state, response, recordedExposures), modelResolution: modelResolution(config, response), promptSha256: promptSha256(request), providerRequestSha256: providerRequestSha256(state.invocations), @@ -510,6 +531,7 @@ function localStageMetadata( requestedModel: config.model, returnedProvider: null, returnedModel: null, + providerExposures: providerExposures(state), modelResolution: 'local', promptSha256: promptSha256(request), providerRequestSha256: providerRequestSha256(state.invocations), @@ -540,6 +562,7 @@ function notRunStageMetadata(trace: FixedTraceCase): FixedTraceModelStageMetadat requestedModel: null, returnedProvider: null, returnedModel: null, + providerExposures: Object.freeze([]), modelResolution: null, promptSha256: null, providerRequestSha256: null, @@ -592,6 +615,21 @@ function resolveTraceDefinitions( }); } +/** + * Routed replay currently obtains the presented surface from case fixtures. + * That is useful for deterministic component replay, but it is not neutral + * common-universe provenance and therefore cannot support architecture + * comparison. Keep this refusal separate from replay so it cannot be mistaken + * for an admission merely because execution succeeds. + */ +export function assertFixedTraceArchitectureComparisonPrerequisite( + config: Pick, +): never { + if (fixedTraceArchitectureArm(config.architectureArm).id === "direct_generation") + throw new Error("direct architecture comparison is not admitted without signed capture/source/thread/request binding"); + throw new Error("architecture comparison is not admitted: common authenticated base registry/schema/receipt tool universe is unavailable; fixture-local tool definitions are replay-only"); +} + export function fixedTraceToolSchemaSha256( traceSuite: readonly FixedTraceCase[], definitions: readonly AddieTool[], @@ -1107,6 +1145,7 @@ export async function runFixedTraceCase( result.response, result.usage, state, + result.providerExposures, ); const terminalStatus = terminalStatusForFinishReason(result.response.finishReason, result.text); return { diff --git a/server/src/addie/eval/fixed-trace-safe-snapshot.ts b/server/src/addie/eval/fixed-trace-safe-snapshot.ts new file mode 100644 index 0000000000..875c05a955 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-safe-snapshot.ts @@ -0,0 +1,90 @@ +import { types } from 'node:util'; + +/** + * Detach hostile JSON-shaped input without ever reading a value through the + * object. `structuredClone` is intentionally not used here: it invokes + * getters before it rejects them. Node exposes proxy identity without + * invoking user traps, which lets this boundary fail before reflection. + */ +export function snapshotFixedTraceJson(value: unknown, label: string): unknown { + // Track only the active ancestry: aliases may be copied as separate JSON + // subtrees, while an actual cycle has no JSON representation and must fail + // before recursion can exhaust the stack. + const activeAncestors = new WeakSet(); + const copy = (candidate: unknown, path: string): unknown => { + if (candidate === null || typeof candidate === 'string' || typeof candidate === 'boolean') return candidate; + if (typeof candidate === 'number') { + if (!Number.isFinite(candidate)) throw new Error(`${path} contains a non-finite number`); + return candidate; + } + if (typeof candidate !== 'object') throw new Error(`${path} is not JSON data`); + if (types.isProxy(candidate)) throw new Error(`${path} must not contain a Proxy`); + if (activeAncestors.has(candidate)) throw new Error(`${path} must not contain a cycle`); + activeAncestors.add(candidate); + + try { + if (Array.isArray(candidate)) { + if (Object.getPrototypeOf(candidate) !== Array.prototype || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain array without symbols`); + } + const descriptors = Object.getOwnPropertyDescriptors(candidate) as Record; + const lengthDescriptor = descriptors['length']; + if (!lengthDescriptor || !('value' in lengthDescriptor) || lengthDescriptor.enumerable || !Number.isSafeInteger(lengthDescriptor.value) || lengthDescriptor.value < 0) { + throw new Error(`${path} has an invalid array length descriptor`); + } + const length = lengthDescriptor.value as number; + const output: unknown[] = []; + for (const key of Object.keys(descriptors)) { + if (key === 'length') continue; + if (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= length) { + throw new Error(`${path} contains an extra array property`); + } + } + for (let index = 0; index < length; index += 1) { + const descriptor = descriptors[String(index)]; + if (!descriptor || !('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}[${index}] must be an own enumerable data property`); + } + output.push(copy(descriptor.value, `${path}[${index}]`)); + } + return output; + } + + // Null-prototype records are this membrane's own detached output and + // are also safe to snapshot again at composed plan/ledger boundaries. + const prototype = Object.getPrototypeOf(candidate); + if ((prototype !== Object.prototype && prototype !== null) || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain object without symbols`); + } + const descriptors = Object.getOwnPropertyDescriptors(candidate); + // A null prototype makes __proto__ ordinary JSON data. Defining each + // key also avoids every inherited setter, so it cannot disappear or + // change the detached record's prototype before exact-key validation. + const output = Object.create(null) as Record; + for (const [key, descriptor] of Object.entries(descriptors)) { + if (!('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}.${key} must be an own enumerable data property`); + } + Object.defineProperty(output, key, { + value: copy(descriptor.value, `${path}.${key}`), + enumerable: true, + configurable: true, + writable: true, + }); + } + return output; + } finally { + activeAncestors.delete(candidate); + } + }; + return deepFreezeFixedTrace(copy(value, label)); +} + +/** Freeze only detached JSON data, never an object supplied by a caller. */ +export function deepFreezeFixedTrace(value: T): T { + if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; + for (const descriptor of Object.values(Object.getOwnPropertyDescriptors(value))) { + if ('value' in descriptor) deepFreezeFixedTrace(descriptor.value); + } + return Object.freeze(value); +} diff --git a/server/src/addie/eval/fixed-trace-suite.ts b/server/src/addie/eval/fixed-trace-suite.ts index 7cee007a7f..cce17d6b77 100644 --- a/server/src/addie/eval/fixed-trace-suite.ts +++ b/server/src/addie/eval/fixed-trace-suite.ts @@ -305,6 +305,14 @@ export interface FixedTraceModelStageMetadata { requestedModel: string | null; returnedProvider: ModelProviderId | null; returnedModel: string | null; + /** Identity-only ledger of every prepared attempt; never contains payloads. */ + providerExposures?: readonly { + attempt: number; + preparedProvider: ModelProviderId; + preparedModel: string; + returnedProvider: ModelProviderId | null; + returnedModel: string | null; + }[]; modelResolution: 'exact' | 'provider_canonicalized' | 'local' | null; promptSha256: string | null; providerRequestSha256: string | null; diff --git a/server/src/addie/eval/fixed-trace-tool-loop.ts b/server/src/addie/eval/fixed-trace-tool-loop.ts index 20c11fab88..178d33920b 100644 --- a/server/src/addie/eval/fixed-trace-tool-loop.ts +++ b/server/src/addie/eval/fixed-trace-tool-loop.ts @@ -64,6 +64,14 @@ export interface FixedTraceToolLoopResult { usage: ModelUsage; tools: ReadonlyArray; invocations: ReadonlyArray; + /** Identity-only record for each dispatched model turn; never prompt data. */ + providerExposures: ReadonlyArray<{ + attempt: number; + preparedProvider: PreparedModelInvocation['provider']; + preparedModel: string; + returnedProvider: ModelResponse['provider']; + returnedModel: string; + }>; } export interface FixedTraceToolLoopOptions { @@ -291,6 +299,7 @@ export async function executeFixedTraceToolLoop( const executions: FixedTraceToolExecution[] = []; const completedExecutions: ToolExecution[] = []; const invocations: PreparedModelInvocation[] = []; + const providerExposures: FixedTraceToolLoopResult['providerExposures'][number][] = []; const seenCallIds = new Set(); const seenToolNames = new Set(); const modelLoop = new ModelTurnLoopState(iterationLimit); @@ -328,6 +337,15 @@ export async function executeFixedTraceToolLoop( await options.beforeDispatch?.(prepared); }, }); + const prepared = invocations.at(-1); + if (!prepared) throw new Error('fixed-trace model response was not preceded by a prepared invocation'); + providerExposures.push(Object.freeze({ + attempt: invocations.length, + preparedProvider: prepared.provider, + preparedModel: prepared.model, + returnedProvider: response.provider, + returnedModel: response.model, + })); const turn = activeTurn.acceptResponse(response); if (turn.providerToolCalls.length > 0 || turn.providerToolResults.length > 0) { @@ -350,6 +368,7 @@ export async function executeFixedTraceToolLoop( usage: modelLoop.usage, tools: Object.freeze([...executions]), invocations: Object.freeze([...invocations]), + providerExposures: Object.freeze([...providerExposures]), }; } diff --git a/server/src/addie/model-cost-pricing.ts b/server/src/addie/model-cost-pricing.ts index 5397fdeca1..75b1b031ef 100644 --- a/server/src/addie/model-cost-pricing.ts +++ b/server/src/addie/model-cost-pricing.ts @@ -10,15 +10,45 @@ import { CLAUDE_PRICING_VERSION, costUsdMicros, resolveKnownClaudePricingModel, -} from './claude-pricing.js'; +} from "./claude-pricing.js"; import { GOOGLE_ROUTER_MODEL, isGoogleRouterModelRevision, -} from './model-providers/google-generate-content-provider.js'; -import type { ModelProviderId, ModelUsage } from './model-providers/model-provider.js'; +} from "./model-providers/google-generate-content-provider.js"; +import { OPENAI_ROUTER_MODEL } from "./model-providers/openai-responses-provider.js"; +import type { + ModelProviderId, + ModelUsage, +} from "./model-providers/model-provider.js"; export const GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION = - 'google-gemini-3.7-flash-through-2026-12-31' as const; + "google-gemini-3.7-flash-through-2026-12-31" as const; + +/** + * This is the existing, reviewed Luna router price identity used by the + * shadow/canary controls. Keep it literal: Terra and Sol have no adapter or + * reviewed price entry and must not inherit Luna's availability or rate. + */ +export const OPENAI_GPT_5_6_LUNA_PRICING_VERSION = + "openai-gpt-5.6-luna-2026-08-26" as const; + +/** + * The single reviewed Luna price identity. Fixed-trace planning, reservation, + * and settlement import this record rather than copying a near-match profile. + * `inputTokens` includes cached input, so cached input is a subset replacement + * (not an additional charge and not an uncached charge). + */ +export const OPENAI_GPT_5_6_LUNA_PRICING = Object.freeze({ + profileId: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, + inputUsdPerMillionTokens: 0.2, + outputUsdPerMillionTokens: 1.2, + cacheReadUsdPerMillionTokens: 0.02, + cacheReadAccounting: "subset" as const, + cacheWriteUsdPerMillionTokens: null, + cacheWriteAccounting: "unsupported" as const, + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", +}); export interface ModelCostPricing { provider: ModelProviderId; @@ -30,14 +60,18 @@ export interface ModelCostPricing { /** A complete provider-normalized usage tuple is required for live charging. */ export function hasCompleteModelUsage(usage: unknown): usage is ModelUsage { - if (!usage || typeof usage !== 'object') return false; + if (!usage || typeof usage !== "object") return false; const value = usage as Record; const isSafeCount = (count: unknown): count is number => - typeof count === 'number' && Number.isSafeInteger(count) && count >= 0; - return isSafeCount(value.inputTokens) - && isSafeCount(value.outputTokens) - && (value.cacheReadTokens === undefined || isSafeCount(value.cacheReadTokens)) - && (value.cacheWriteTokens === undefined || isSafeCount(value.cacheWriteTokens)); + typeof count === "number" && Number.isSafeInteger(count) && count >= 0; + return ( + isSafeCount(value.inputTokens) && + isSafeCount(value.outputTokens) && + (value.cacheReadTokens === undefined || + isSafeCount(value.cacheReadTokens)) && + (value.cacheWriteTokens === undefined || + isSafeCount(value.cacheWriteTokens)) + ); } /** @@ -50,37 +84,60 @@ export function resolveModelCostPricing( provider: ModelProviderId | string, model: string, ): ModelCostPricing | null { - const canonicalAnthropicModel = provider === 'anthropic' - ? resolveKnownClaudePricingModel(model) - : null; - if (provider === 'anthropic' && canonicalAnthropicModel) { + if (provider === "openai" && model === OPENAI_ROUTER_MODEL) { + return { + provider: "openai", + model: OPENAI_ROUTER_MODEL, + version: OPENAI_GPT_5_6_LUNA_PRICING_VERSION, + validBefore: null, + estimateCostMicros: (usage) => { + const cacheReadTokens = usage.cacheReadTokens ?? 0; + const uncachedInputTokens = + cacheReadTokens <= usage.inputTokens + ? usage.inputTokens - cacheReadTokens + : usage.inputTokens; + return Math.ceil( + uncachedInputTokens * + OPENAI_GPT_5_6_LUNA_PRICING.inputUsdPerMillionTokens + + cacheReadTokens * + OPENAI_GPT_5_6_LUNA_PRICING.cacheReadUsdPerMillionTokens + + usage.outputTokens * + OPENAI_GPT_5_6_LUNA_PRICING.outputUsdPerMillionTokens, + ); + }, + }; + } + const canonicalAnthropicModel = + provider === "anthropic" ? resolveKnownClaudePricingModel(model) : null; + if (provider === "anthropic" && canonicalAnthropicModel) { return { - provider: 'anthropic', + provider: "anthropic", model, version: `${CLAUDE_PRICING_VERSION}:${canonicalAnthropicModel}`, validBefore: null, - estimateCostMicros: (usage) => costUsdMicros(canonicalAnthropicModel, { - input_tokens: usage.inputTokens, - output_tokens: usage.outputTokens, - cache_read_input_tokens: usage.cacheReadTokens, - cache_creation_input_tokens: usage.cacheWriteTokens, - }), + estimateCostMicros: (usage) => + costUsdMicros(canonicalAnthropicModel, { + input_tokens: usage.inputTokens, + output_tokens: usage.outputTokens, + cache_read_input_tokens: usage.cacheReadTokens, + cache_creation_input_tokens: usage.cacheWriteTokens, + }), }; } // Google Generate Content accepts this canonical router model and its // provider-returned eight-digit dated revisions (for example `...-20260801`). Keep this // mapping here, beside the reviewed rate, rather than falling back to any // other model or provider price. - const canonicalGoogleModel = provider === 'google' - && isGoogleRouterModelRevision(model) - ? GOOGLE_ROUTER_MODEL - : null; + const canonicalGoogleModel = + provider === "google" && isGoogleRouterModelRevision(model) + ? GOOGLE_ROUTER_MODEL + : null; if (canonicalGoogleModel) { return { - provider: 'google', + provider: "google", model, version: GOOGLE_GEMINI_3_7_FLASH_PRICING_VERSION, - validBefore: new Date('2027-01-01T00:00:00.000Z'), + validBefore: new Date("2027-01-01T00:00:00.000Z"), // Official standard pricing checked 2026-08-30: $0.75/M input, // $0.075/M cached input, and $3.75/M output (including thought tokens). // A cache-read count above input is charged in addition to all input, @@ -88,14 +145,15 @@ export function resolveModelCostPricing( estimateCostMicros: (usage) => { const cacheReadTokens = usage.cacheReadTokens ?? 0; const cacheWriteTokens = usage.cacheWriteTokens ?? 0; - const uncachedInput = cacheReadTokens <= usage.inputTokens - ? usage.inputTokens - cacheReadTokens - : usage.inputTokens; + const uncachedInput = + cacheReadTokens <= usage.inputTokens + ? usage.inputTokens - cacheReadTokens + : usage.inputTokens; return Math.ceil( - uncachedInput * 0.75 - + cacheReadTokens * 0.075 - + cacheWriteTokens * 0.75 - + usage.outputTokens * 3.75, + uncachedInput * 0.75 + + cacheReadTokens * 0.075 + + cacheWriteTokens * 0.75 + + usage.outputTokens * 3.75, ); }, }; diff --git a/server/tests/manual/fixed-trace-provider-eval.ts b/server/tests/manual/fixed-trace-provider-eval.ts index e385a72cf1..7f7d046cd5 100644 --- a/server/tests/manual/fixed-trace-provider-eval.ts +++ b/server/tests/manual/fixed-trace-provider-eval.ts @@ -1,362 +1,36 @@ -/** - * Live synthetic fixed-trace replay across normalized providers. - * - * Production handlers and production messages are never loaded into the - * executor: every tool result comes from the immutable fixed-trace fixtures. - * The required shared soft budget admits each exact prepared request before - * dispatch and halts after unknown spend exposure. - * - * `--architecture-arm=direct_generation` is intentionally admission-only. - * Production builds an authorization-aware definition/handler intersection - * before intent narrowing, but this harness neither captures that intersection - * nor bounds it independently; fixture-local schemas must not stand in for it. - * `oracle_route_diagnostic` may execute generation with fixture routing. - * The hybrid-only `--suite=hybrid-evaluator` binds the separately reviewed - * local-admission corpus without altering the legacy 32 traces. Every arm is - * diagnostic-only in this foundation: independent judging, - * comparison, and rollout are blocked until an evaluator-owned run-context - * and raw-ledger coordinator can authenticate serialized artifacts. - * - * Example: - * DOTENV_CONFIG_PATH=.env.local npm run eval:addie-fixed-traces -- \ - * --soft-max-usd=1 --output=.context/evals/fixed-traces.json - */ -import { createHash, randomUUID } from 'node:crypto'; -import { execFileSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { ModelConfig } from '../../src/config/models.js'; -import { CODE_VERSION, computeRouterRulesHash } from '../../src/addie/config-version.js'; -import { - BudgetedFixedTraceProvider, - FixedTraceBudget, - fixedTraceResponsePricingPolicy, -} from '../../src/addie/eval/fixed-trace-budget.js'; -import { - type FixedTraceProviderStageConfig, -} from '../../src/addie/eval/fixed-trace-runner.js'; -import { - runFixedTraceDiagnosticArtifact, - type FixedTraceDiagnosticProviderPlan, -} from '../../src/addie/eval/fixed-trace-diagnostic-run.js'; -import { MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS } from '../../src/addie/eval/fixed-trace-tool-loop.js'; -import { parseFixedTraceDiagnosticCliArguments } from '../../src/addie/eval/fixed-trace-diagnostic-cli.js'; -import { reserveFixedTraceDiagnosticOutput } from '../../src/addie/eval/fixed-trace-diagnostic-output.js'; -import { canonicalFixedTraceToolDefinitions } from '../../src/addie/eval/fixed-trace-tools.js'; -import { - fixedTraceHybridPolicy, - type FixedTraceArchitectureArmId, -} from '../../src/addie/eval/fixed-trace-architecture.js'; -import { - FIXED_TRACE_SUITE, - FIXED_TRACE_HYBRID_EVALUATOR_SUITE, - fixedTraceSuiteSha256, - type FixedTracePricing, -} from '../../src/addie/eval/fixed-trace-suite.js'; -import { AnthropicRouterProvider } from '../../src/addie/model-providers/anthropic-router-provider.js'; -import { AnthropicModelProvider } from '../../src/addie/model-providers/anthropic-provider.js'; -import type { - ModelProvider, - ModelProviderId, - ModelReasoningEffort, -} from '../../src/addie/model-providers/model-provider.js'; -import { - OpenAIResponsesProvider, - OPENAI_ROUTER_MODEL, -} from '../../src/addie/model-providers/openai-responses-provider.js'; -import { - GoogleGenerateContentProvider, - GOOGLE_ROUTER_MODEL, -} from '../../src/addie/model-providers/google-generate-content-provider.js'; -import { loadResponseStyle, loadRules } from '../../src/addie/rules/index.js'; +/** Planning-only manual entrypoint: it has no dispatch or output path. */ +import { parseFixedTraceDiagnosticCliArguments } from "../../src/addie/eval/fixed-trace-diagnostic-cli.js"; -type ProviderName = ModelProviderId; - -type ProviderPlan = FixedTraceDiagnosticProviderPlan & { name: ProviderName }; - -const PRICING = { - anthropicRouter: { - profileId: 'anthropic-standard-2026-08:claude-haiku-4-5', - inputUsdPerMillionTokens: 1, - outputUsdPerMillionTokens: 5, - cacheReadUsdPerMillionTokens: 0.1, - cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.', - }, - anthropicGeneration: { - profileId: 'anthropic-standard-2026-08:claude-sonnet-5', - inputUsdPerMillionTokens: 3, - outputUsdPerMillionTokens: 15, - cacheReadUsdPerMillionTokens: 0.3, - cacheWriteUsdPerMillionTokens: 3.75, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Sonnet 5 standard, refreshed August 2026.', - }, - openai: { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', - inputUsdPerMillionTokens: 0.2, - outputUsdPerMillionTokens: 1.2, - cacheReadUsdPerMillionTokens: 0.02, - cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'unsupported', - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', - }, - google: { - profileId: 'google-gemini-3.7-flash-through-2026-12-31', - inputUsdPerMillionTokens: 0.75, - outputUsdPerMillionTokens: 3.75, - cacheReadUsdPerMillionTokens: 0.075, - cacheWriteUsdPerMillionTokens: 0.75, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'additive', - source: 'Google Gemini 3.7 Flash introductory standard, checked 2026-08-25.', - }, -} satisfies Record; - -const cliArguments = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); - -function argument(name: string): string | undefined { - return cliArguments[{ providers: 'providers', 'architecture-arm': 'architectureArm', suite: 'suite', 'soft-max-usd': 'softMaxUsd', output: 'output' }[name] as keyof typeof cliArguments] as string | undefined; -} - -function sha256(value: string): string { - return createHash('sha256').update(value, 'utf8').digest('hex'); -} - -function sourceBundle(): { sha256: string; files: string[] } { - const trackedFiles = execFileSync('git', [ - 'ls-files', '-z', 'package.json', 'package-lock.json', 'server/src/addie', - 'server/src/config/models.ts', 'server/tests/manual/fixed-trace-provider-eval.ts', - ], { encoding: 'utf8' }).split('\0').filter(Boolean).sort(); - const files = [...new Set([ - ...trackedFiles, - 'server/src/addie/eval/fixed-trace-budget.ts', - 'server/src/addie/eval/fixed-trace-architecture.ts', - 'server/src/addie/eval/fixed-trace-runner.ts', - 'server/tests/manual/fixed-trace-provider-eval.ts', - ])].sort(); - const hash = createHash('sha256'); - for (const file of files) { - hash.update(file, 'utf8').update('\0').update(readFileSync(file)).update('\0'); - } - return { sha256: hash.digest('hex'), files }; -} - -function stage( - provider: ModelProvider, - model: string, - reasoningEffort: ModelReasoningEffort, - maxOutputTokens: number, - maxIterations: number, - pricing: FixedTracePricing, -): FixedTraceProviderStageConfig { - return { - provider, - model, - reasoningEffort, - maxOutputTokens, - timeoutMs: 120_000, - maxIterations, - transportRetries: 0, - samplingMode: 'provider_no_sampling_control', - temperature: null, - pricing, - }; -} - -function budgetedStageProvider( - provider: ModelProvider, - budget: FixedTraceBudget, - model: string, - pricing: FixedTracePricing, -): BudgetedFixedTraceProvider { - return new BudgetedFixedTraceProvider( - provider, - budget, - pricing, - fixedTraceResponsePricingPolicy(provider.id, model, pricing), +const arguments_ = parseFixedTraceDiagnosticCliArguments(process.argv.slice(2)); +if (!arguments_.validateOnly) + throw new Error( + "This planning-only evaluator requires --validate-only and cannot dispatch providers", ); -} - -function providerPlans( - names: readonly ProviderName[], - budget: FixedTraceBudget, -): ProviderPlan[] { - const plans: ProviderPlan[] = []; - if (names.includes('anthropic')) { - if (!process.env.ANTHROPIC_API_KEY) throw new Error('ANTHROPIC_API_KEY is required'); - if (ModelConfig.fast !== 'claude-haiku-4-5') throw new Error('Fixed traces pin Anthropic routing to claude-haiku-4-5'); - if (ModelConfig.primary !== 'claude-sonnet-5') throw new Error('Fixed traces pin Anthropic generation to claude-sonnet-5'); - const router = new AnthropicRouterProvider(process.env.ANTHROPIC_API_KEY, { maxRetries: 0 }); - const generation = new AnthropicModelProvider( - process.env.ANTHROPIC_API_KEY, - undefined, - { transportMaxRetries: 0 }, - ); - const budgetedGeneration = budgetedStageProvider(generation, budget, ModelConfig.primary, PRICING.anthropicGeneration); - plans.push({ - name: 'anthropic', - router: stage( - budgetedStageProvider(router, budget, ModelConfig.fast, PRICING.anthropicRouter), - ModelConfig.fast, - 'provider_default', - 300, - 1, - PRICING.anthropicRouter, - ), - generation: stage( - budgetedGeneration, - ModelConfig.primary, - 'provider_default', - 900, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.anthropicGeneration, - ), - }); - } - if (names.includes('openai')) { - if (!process.env.OPENAI_API_KEY) throw new Error('OPENAI_API_KEY is required'); - const provider = new OpenAIResponsesProvider(process.env.OPENAI_API_KEY); - const budgetedProvider = budgetedStageProvider(provider, budget, OPENAI_ROUTER_MODEL, PRICING.openai); - plans.push({ - name: 'openai', - router: stage( - budgetedProvider, - OPENAI_ROUTER_MODEL, - 'none', - 300, - 1, - PRICING.openai, - ), - generation: stage( - budgetedProvider, - OPENAI_ROUTER_MODEL, - 'none', - 900, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.openai, - ), - }); - } - if (names.includes('google')) { - if (!process.env.GEMINI_API_KEY) throw new Error('GEMINI_API_KEY is required'); - const provider = new GoogleGenerateContentProvider(process.env.GEMINI_API_KEY); - const budgetedProvider = budgetedStageProvider(provider, budget, GOOGLE_ROUTER_MODEL, PRICING.google); - plans.push({ - name: 'google', - router: stage( - budgetedProvider, - GOOGLE_ROUTER_MODEL, - 'low', - 1_200, - 1, - PRICING.google, - ), - generation: stage( - budgetedProvider, - GOOGLE_ROUTER_MODEL, - 'low', - 1_200, - MAX_FIXED_TRACE_TOOL_LOOP_ITERATIONS, - PRICING.google, - ), - }); - } - return plans; -} - -const providerNames = (argument('providers') ?? 'anthropic,openai,google').split(',') as ProviderName[]; -if (providerNames.some((name) => !['anthropic', 'openai', 'google'].includes(name))) { - throw new Error('Unknown --providers value'); -} -if (new Set(providerNames).size !== providerNames.length || providerNames.length === 0) { - throw new Error('--providers must contain one or more unique providers'); -} -const architectureArm = (argument('architecture-arm') ?? 'two_stage_llm_router') as FixedTraceArchitectureArmId; -if (!(architectureArm in { two_stage_llm_router: true, direct_generation: true, deterministic_policy_llm_fallback_hybrid: true, oracle_route_diagnostic: true })) { - throw new Error('Unknown --architecture-arm value'); -} -const suiteName = argument('suite') ?? 'canonical'; -if (suiteName !== 'canonical' && suiteName !== 'hybrid-evaluator') throw new Error('Unknown --suite value'); -if (suiteName === 'hybrid-evaluator' && architectureArm !== 'deterministic_policy_llm_fallback_hybrid') { - throw new Error('--suite=hybrid-evaluator requires --architecture-arm=deterministic_policy_llm_fallback_hybrid'); -} -const traceSuite = suiteName === 'hybrid-evaluator' - ? FIXED_TRACE_HYBRID_EVALUATOR_SUITE - : FIXED_TRACE_SUITE; -const softMaxUsd = Number(argument('soft-max-usd')); -if (!Number.isFinite(softMaxUsd) || softMaxUsd <= 0) { - throw new Error('--soft-max-usd is required and must be positive'); -} -const outputArgument = argument('output'); -if (!outputArgument?.trim()) throw new Error('--output is required'); -const outputPath = resolve(outputArgument); -if (cliArguments.validateOnly) { - console.log(JSON.stringify({ +if (arguments_.output !== undefined) + throw new Error( + "--output is unavailable in validate-only mode; no artifact may be written", + ); +// This entrypoint is deliberately data-only. Some transitive corpus modules +// initialize diagnostic loggers while their immutable declarations load; keep +// those process-local diagnostics isolated from the one-machine-readable-line +// validate-only contract. +process.env.LOG_LEVEL = "silent"; +const { + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + assertFixedTraceEvaluationProtocol, + estimateFixedTraceEvaluationProtocol, +} = await import("../../src/addie/eval/fixed-trace-evaluation-protocol.js"); +assertFixedTraceEvaluationProtocol(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); +const estimate = estimateFixedTraceEvaluationProtocol( + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, +); +console.log( + JSON.stringify({ diagnosticOnly: true, - judgeDispatch: 'blocked_pending_trusted_evaluator_owned_coordinator', - validated: { - providers: providerNames, - architectureArm, - suite: suiteName, - softMaxUsd, - outputPath, - }, - })); - process.exit(0); -} -// This exclusive create happens before source inspection, credentials, -// provider construction, or dispatch. Never unlink it: an empty file is the -// truthful crash/incomplete marker if later setup fails. -const outputReservation = reserveFixedTraceDiagnosticOutput(outputPath); - -const gitCommit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); -const gitDirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim().length > 0; -const sources = sourceBundle(); -const promptConfigVersion = sha256(JSON.stringify({ - codeVersion: CODE_VERSION, - routerRulesHash: computeRouterRulesHash(), - rules: loadRules(), - responseStyle: loadResponseStyle(), -})); -const toolDefinitions = canonicalFixedTraceToolDefinitions(traceSuite); -const budget = new FixedTraceBudget(softMaxUsd); -const plans = providerPlans(providerNames, budget); -const runStartedAt = new Date().toISOString(); -const runRootId = `fixed-trace-${runStartedAt}-${randomUUID()}`; -const artifact = await runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: sources.sha256, - gitCommit, - gitDirty, - promptConfigVersion, - traceSuite, - traceSuiteSha256: fixedTraceSuiteSha256(traceSuite), - toolDefinitions, - toolDefinitionProvenance: 'fixture_local', - architectureArm, - ...(architectureArm === 'deterministic_policy_llm_fallback_hybrid' - ? { hybridPolicy: fixedTraceHybridPolicy() } - : {}), - }, - budget, - outputReservation, - runRootId, - runStartedAt, - sourceBundleFiles: sources.files, - budgetNote: 'Soft admission target: exact prepared-request bytes and the full output allowance are reserved before each dispatch. Remote work may continue after a client timeout; any dispatched call without terminal usage marks exposure unknown and blocks every later dispatch.', -}); -console.log(JSON.stringify({ - outputPath, - runRootId, - providers: providerNames, - suite: suiteName, - comparisonEligible: artifact.comparisonEligible, - rolloutPass: artifact.rolloutPass, - budget: artifact.budget, -}, null, 2)); + dispatchable: false, + outputWritten: false, + providerCalls: 0, + externalFinalN: estimate.externalFinalN, + totalCeilingUsd: estimate.totalCeilingUsd, + }), +); diff --git a/server/tests/unit/addie/direct-tool-universe.test.ts b/server/tests/unit/addie/direct-tool-universe.test.ts index 3d708ea5c5..e1f0358c4f 100644 --- a/server/tests/unit/addie/direct-tool-universe.test.ts +++ b/server/tests/unit/addie/direct-tool-universe.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { createSyntheticDirectToolReceiptHandlers, FIXED_TRACE_DIRECT_TOOL_UNIVERSE, + fixedTraceDirectToolHandlers, type CapturedDirectToolUniverse, } from '../../../src/addie/direct-tool-universe.js'; import { getSafeReadOnlyFallbackTools } from '../../../src/addie/tool-sets.js'; @@ -53,4 +54,9 @@ describe('direct tool-universe evaluator descriptors', () => { }); expect(mockHandler).not.toHaveBeenCalled(); }); + + it('constructs synthetic handlers only through the explicit replay factory', () => { + const handlers = fixedTraceDirectToolHandlers(); + expect([...handlers.keys()]).toEqual(FIXED_TRACE_DIRECT_TOOL_UNIVERSE.toolNames); + }); }); diff --git a/server/tests/unit/addie/fixed-trace-budget.test.ts b/server/tests/unit/addie/fixed-trace-budget.test.ts index bc3be356e3..732465c97c 100644 --- a/server/tests/unit/addie/fixed-trace-budget.test.ts +++ b/server/tests/unit/addie/fixed-trace-budget.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it, vi } from "vitest"; import { BudgetedFixedTraceProvider, FixedTraceBudget, @@ -6,8 +6,8 @@ import { fixedTraceEstimatedCostUsd, fixedTraceApprovedPricingProfiles, fixedTraceResponsePricingPolicy, -} from '../../../src/addie/eval/fixed-trace-budget.js'; -import { collectModelResponse } from '../../../src/addie/model-providers/events.js'; +} from "../../../src/addie/eval/fixed-trace-budget.js"; +import { collectModelResponse } from "../../../src/addie/model-providers/events.js"; import type { ModelProvider, ModelProviderCapabilities, @@ -16,7 +16,7 @@ import type { ModelResponse, NormalizedModelEvent, PreparedModelInvocation, -} from '../../../src/addie/model-providers/model-provider.js'; +} from "../../../src/addie/model-providers/model-provider.js"; const CAPABILITIES: ModelProviderCapabilities = { streaming: false, @@ -30,42 +30,45 @@ const CAPABILITIES: ModelProviderCapabilities = { }; const REQUEST: ModelRequest = { - model: 'gpt-5.6-luna', + model: "gpt-5.6-luna", system: [], - messages: [{ role: 'user', content: [{ type: 'text', text: 'Synthetic request.' }] }], + messages: [ + { role: "user", content: [{ type: "text", text: "Synthetic request." }] }, + ], tools: [], maxOutputTokens: 100, }; const RESPONSE: ModelResponse = { - provider: 'openai', - model: 'gpt-5.6-luna', - id: 'response-1', - content: [{ type: 'text', text: 'Synthetic response.' }], - finishReason: 'stop', - providerFinishReason: 'completed', + provider: "openai", + model: "gpt-5.6-luna", + id: "response-1", + content: [{ type: "text", text: "Synthetic response." }], + finishReason: "stop", + providerFinishReason: "completed", usage: { inputTokens: 10, outputTokens: 5 }, }; const PRICING = { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', + profileId: "openai-gpt-5.6-luna-2026-08-26", inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset' as const, - cacheWriteAccounting: 'unsupported' as const, - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', + cacheReadAccounting: "subset" as const, + cacheWriteAccounting: "unsupported" as const, + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", }; const RESPONSE_PRICING_POLICY = fixedTraceResponsePricingPolicy( - 'openai', - 'gpt-5.6-luna', + "openai", + "gpt-5.6-luna", PRICING, ); class BudgetScriptedProvider implements ModelProvider { - readonly id = 'openai' as const; + readonly id = "openai" as const; readonly capabilities = CAPABILITIES; readonly dispatches = vi.fn(); @@ -87,101 +90,144 @@ class BudgetScriptedProvider implements ModelProvider { await options.beforeDispatch?.(this.prepare(request)); this.dispatches(); const next = this.script.shift(); - if (!next) throw new Error('Script exhausted'); + if (!next) throw new Error("Script exhausted"); if (next instanceof Error) throw next; - yield { type: 'response_start', provider: this.id, model: next.model, id: next.id }; - yield { type: 'text_delta', index: 0, text: 'Synthetic response.' }; - yield { type: 'response_complete', response: next }; + yield { + type: "response_start", + provider: this.id, + model: next.model, + id: next.id, + }; + yield { type: "text_delta", index: 0, text: "Synthetic response." }; + yield { type: "response_complete", response: next }; } } -describe('fixed trace provider budget', () => { - it('exposes only reviewed production pricing and rejects former test profiles before dispatch', () => { +describe("fixed trace provider budget", () => { + it("exposes only reviewed production pricing and rejects former test profiles before dispatch", () => { const liveProfiles = fixedTraceApprovedPricingProfiles(); expect(liveProfiles).toHaveLength(4); for (const profile of liveProfiles) { - expect(`${profile.expectedModel}\n${profile.profileId}\n${profile.source}`).not.toMatch(/synthetic|test/i); + expect( + `${profile.expectedModel}\n${profile.profileId}\n${profile.source}`, + ).not.toMatch(/synthetic|test/i); } const delegate = new BudgetScriptedProvider([RESPONSE]); - expect(() => fixedTraceResponsePricingPolicy('anthropic', 'synthetic-manual-model', { - profileId: 'synthetic-manual-artifact-v1', - inputUsdPerMillionTokens: 1, - outputUsdPerMillionTokens: 5, - cacheReadUsdPerMillionTokens: null, - cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'unsupported', - cacheWriteAccounting: 'unsupported', - source: 'Synthetic manual artifact pricing.', - })).toThrow('Fixed trace pricing profile is not evaluator approved'); + expect(() => + fixedTraceResponsePricingPolicy("anthropic", "synthetic-manual-model", { + profileId: "synthetic-manual-artifact-v1", + inputUsdPerMillionTokens: 1, + outputUsdPerMillionTokens: 5, + cacheReadUsdPerMillionTokens: null, + cacheWriteUsdPerMillionTokens: null, + cacheReadAccounting: "unsupported", + cacheWriteAccounting: "unsupported", + source: "Synthetic manual artifact pricing.", + }), + ).toThrow("Fixed trace pricing profile is not evaluator approved"); expect(delegate.dispatches).not.toHaveBeenCalled(); }); - it('prices Google-style subset reads plus additive writes explicitly', () => { - expect(fixedTraceEstimatedCostUsd({ inputTokens: 100, outputTokens: 10, cacheReadTokens: 40, cacheWriteTokens: 20 }, { - ...PRICING, - cacheReadUsdPerMillionTokens: 0.5, - cacheWriteUsdPerMillionTokens: 1, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'additive', - })).toBeCloseTo(0.00015); + it("prices Google-style subset reads plus additive writes explicitly", () => { + expect( + fixedTraceEstimatedCostUsd( + { + inputTokens: 100, + outputTokens: 10, + cacheReadTokens: 40, + cacheWriteTokens: 20, + }, + { + ...PRICING, + cacheReadUsdPerMillionTokens: 0.5, + cacheWriteUsdPerMillionTokens: 1, + cacheReadAccounting: "subset", + cacheWriteAccounting: "additive", + }, + ), + ).toBeCloseTo(0.00015); }); - it('prices additive Anthropic cache buckets without treating them as input subsets', () => { - expect(fixedTraceEstimatedCostUsd({ - inputTokens: 10, - outputTokens: 0, - cacheReadTokens: 20, - cacheWriteTokens: 30, - }, { - ...PRICING, - cacheReadUsdPerMillionTokens: 2, - cacheWriteUsdPerMillionTokens: 3, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - })).toBeCloseTo(0.00014); + it("prices additive Anthropic cache buckets without treating them as input subsets", () => { + expect( + fixedTraceEstimatedCostUsd( + { + inputTokens: 10, + outputTokens: 0, + cacheReadTokens: 20, + cacheWriteTokens: 30, + }, + { + ...PRICING, + cacheReadUsdPerMillionTokens: 2, + cacheWriteUsdPerMillionTokens: 3, + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + }, + ), + ).toBeCloseTo(0.00014); }); - it('fails closed when a nonzero cache bucket has no recorded formula', () => { - expect(() => fixedTraceEstimatedCostUsd({ inputTokens: 10, outputTokens: 0, cacheReadTokens: 1 }, { - ...PRICING, - cacheReadUsdPerMillionTokens: null, - cacheReadAccounting: 'unsupported', - })) - .toThrow('cache read accounting is unavailable'); + it("fails closed when a nonzero cache bucket has no recorded formula", () => { + expect(() => + fixedTraceEstimatedCostUsd( + { inputTokens: 10, outputTokens: 0, cacheReadTokens: 1 }, + { + ...PRICING, + cacheReadUsdPerMillionTokens: null, + cacheReadAccounting: "unsupported", + }, + ), + ).toThrow("cache read accounting is unavailable"); }); - it('closes shared admission rather than settling an unapproved returned model at requested rates', async () => { - const mismatched = { ...RESPONSE, model: 'other-openai-model' }; + it("closes shared admission rather than settling an unapproved returned model at requested rates", async () => { + const mismatched = { ...RESPONSE, model: "other-openai-model" }; const delegate = new BudgetScriptedProvider([mismatched, RESPONSE]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).resolves.toEqual(mismatched); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).resolves.toEqual(mismatched); expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0, dispatchedCalls: 1, completedCalls: 0, exposureUnknown: true, }); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toMatchObject({ - name: 'FixedTraceBudgetAdmissionError', reason: 'budget_exposure_unknown', + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toMatchObject({ + name: "FixedTraceBudgetAdmissionError", + reason: "budget_exposure_unknown", }); expect(delegate.dispatches).toHaveBeenCalledTimes(1); }); - it('rejects a caller callback as returned-model pricing authority', () => { + it("rejects a caller callback as returned-model pricing authority", () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(1); - expect(() => new BudgetedFixedTraceProvider( - delegate, - budget, - PRICING, - (() => true) as unknown as typeof RESPONSE_PRICING_POLICY, - )).toThrow('Fixed trace returned-model pricing policy is not evaluator approved'); + expect( + () => + new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + (() => true) as unknown as typeof RESPONSE_PRICING_POLICY, + ), + ).toThrow( + "Fixed trace returned-model pricing policy is not evaluator approved", + ); }); - it('reserves an additive cache-write worst case before dispatch', () => { + it("reserves an additive cache-write worst case before dispatch", () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(10_000); const reservation = budget.reserve(delegate.prepare(REQUEST), 1, { @@ -189,24 +235,31 @@ describe('fixed trace provider budget', () => { outputUsdPerMillionTokens: 0, cacheReadUsdPerMillionTokens: null, cacheWriteUsdPerMillionTokens: 1_000_000, - cacheReadAccounting: 'unsupported', - cacheWriteAccounting: 'additive', - source: 'synthetic additive cache-write worst case', + cacheReadAccounting: "unsupported", + cacheWriteAccounting: "additive", + source: "synthetic additive cache-write worst case", }); // The cache-write rate is deliberately much larger than input. A reserve // that only charged base input would be zero here. expect(budget.snapshot().reservedUsd).toBeGreaterThan(1); budget.cancel(reservation); }); - it('rejects over-budget work before provider dispatch', async () => { + it("rejects over-budget work before provider dispatch", async () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(0.000001); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toMatchObject({ - name: 'FixedTraceBudgetAdmissionError', - reason: 'soft_limit_exceeded', - terminalStatus: 'not_dispatched_budget', + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toMatchObject({ + name: "FixedTraceBudgetAdmissionError", + reason: "soft_limit_exceeded", + terminalStatus: "not_dispatched_budget", }); expect(delegate.dispatches).not.toHaveBeenCalled(); expect(budget.snapshot()).toMatchObject({ @@ -219,12 +272,19 @@ describe('fixed trace provider budget', () => { }); }); - it('releases the reserve and accounts terminal usage', async () => { + it("releases the reserve and accounts terminal usage", async () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).resolves.toEqual(RESPONSE); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).resolves.toEqual(RESPONSE); expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0.000008, reservedUsd: 0, @@ -235,31 +295,46 @@ describe('fixed trace provider budget', () => { }); }); - it('uses one frozen terminal snapshot despite delegate mutation after response_complete', async () => { + it("uses one frozen terminal snapshot despite delegate mutation after response_complete", async () => { const original = structuredClone(RESPONSE); const delegate: ModelProvider = { - id: 'openai', + id: "openai", capabilities: CAPABILITIES, prepare(request): PreparedModelInvocation { return { - provider: 'openai', model: request.model, capabilities: CAPABILITIES, + provider: "openai", + model: request.model, + capabilities: CAPABILITIES, providerRequest: { model: request.model }, }; }, - async *respond(request, options = {}): AsyncIterable { + async *respond( + request, + options = {}, + ): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); - yield { type: 'response_start', provider: 'openai', model: original.model, id: original.id }; - yield { type: 'text_delta', index: 0, text: 'Synthetic response.' }; - yield { type: 'response_complete', response: original }; - original.id = 'mutated-response-id'; - original.model = 'mutated-model'; - original.content[0] = { type: 'text', text: 'Mutated response.' }; + yield { + type: "response_start", + provider: "openai", + model: original.model, + id: original.id, + }; + yield { type: "text_delta", index: 0, text: "Synthetic response." }; + yield { type: "response_complete", response: original }; + original.id = "mutated-response-id"; + original.model = "mutated-model"; + original.content[0] = { type: "text", text: "Mutated response." }; original.usage.inputTokens = 999_999; original.usage.outputTokens = 999_999; }, }; const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); const collected = await collectModelResponse(provider.respond(REQUEST)); @@ -275,35 +350,52 @@ describe('fixed trace provider budget', () => { }); }); - it('halts later calls after a dispatched response has unknown usage', async () => { - const delegate = new BudgetScriptedProvider([new Error('transport failed'), RESPONSE]); + it("halts later calls after a dispatched response has unknown usage", async () => { + const delegate = new BudgetScriptedProvider([ + new Error("transport failed"), + RESPONSE, + ]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toThrow('transport failed'); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toThrow("transport failed"); expect(budget.snapshot()).toMatchObject({ dispatchedCalls: 1, completedCalls: 0, exposureUnknown: true, }); - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toBeInstanceOf( - FixedTraceBudgetAdmissionError, - ); + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toBeInstanceOf(FixedTraceBudgetAdmissionError); expect(delegate.dispatches).toHaveBeenCalledTimes(1); expect(budget.snapshot().budgetRejectedCalls).toBe(1); }); - it('treats malformed terminal usage as unknown exposure', async () => { - const delegate = new BudgetScriptedProvider([{ - ...RESPONSE, - usage: { inputTokens: -1, outputTokens: 5 }, - }]); + it("treats malformed terminal usage as unknown exposure", async () => { + const delegate = new BudgetScriptedProvider([ + { + ...RESPONSE, + usage: { inputTokens: -1, outputTokens: 5 }, + }, + ]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); - - await expect(collectModelResponse(provider.respond(REQUEST))).rejects.toThrow( - 'Fixed trace budget usage is invalid', + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, ); + + await expect( + collectModelResponse(provider.respond(REQUEST)), + ).rejects.toThrow("Fixed trace budget usage is invalid"); expect(budget.snapshot()).toMatchObject({ reservedUsd: 0, remainingUsd: null, @@ -313,14 +405,25 @@ describe('fixed trace provider budget', () => { }); }); - it('does not mark exposure unknown when the caller hook blocks dispatch', async () => { + it("does not mark exposure unknown when the caller hook blocks dispatch", async () => { const delegate = new BudgetScriptedProvider([RESPONSE]); const budget = new FixedTraceBudget(1); - const provider = new BudgetedFixedTraceProvider(delegate, budget, PRICING, RESPONSE_PRICING_POLICY); + const provider = new BudgetedFixedTraceProvider( + delegate, + budget, + PRICING, + RESPONSE_PRICING_POLICY, + ); - await expect(collectModelResponse(provider.respond(REQUEST, { - beforeDispatch: () => { throw new Error('local policy rejected'); }, - }))).rejects.toThrow('local policy rejected'); + await expect( + collectModelResponse( + provider.respond(REQUEST, { + beforeDispatch: () => { + throw new Error("local policy rejected"); + }, + }), + ), + ).rejects.toThrow("local policy rejected"); expect(delegate.dispatches).not.toHaveBeenCalled(); expect(budget.snapshot()).toMatchObject({ reservedUsd: 0, diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts index b98706c109..5459efc9ea 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-cli.test.ts @@ -1,59 +1,82 @@ -import { describe, expect, it } from 'vitest'; -import { execFileSync } from 'node:child_process'; -import { existsSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { parseFixedTraceDiagnosticCliArguments } from '../../../src/addie/eval/fixed-trace-diagnostic-cli.js'; +import { describe, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { parseFixedTraceDiagnosticCliArguments } from "../../../src/addie/eval/fixed-trace-diagnostic-cli.js"; -describe('fixed-trace diagnostic CLI parser', () => { - it('accepts only bounded dry-run forms', () => { - expect(parseFixedTraceDiagnosticCliArguments(['--validate-only', '--providers=openai'])) - .toEqual({ validateOnly: true, providers: 'openai', architectureArm: undefined, suite: undefined, softMaxUsd: undefined, output: undefined }); - expect(parseFixedTraceDiagnosticCliArguments(['--validate-only=true']).validateOnly).toBe(true); +describe("fixed-trace diagnostic CLI parser", () => { + it("accepts only bounded dry-run forms", () => { + expect( + parseFixedTraceDiagnosticCliArguments([ + "--validate-only", + "--providers=openai", + ]), + ).toEqual({ + validateOnly: true, + providers: "openai", + architectureArm: undefined, + suite: undefined, + softMaxUsd: undefined, + output: undefined, + experimentPlan: undefined, + trustedManifest: undefined, + }); + expect( + parseFixedTraceDiagnosticCliArguments(["--validate-only=true"]) + .validateOnly, + ).toBe(true); }); it.each([ - ['--validate-only=false'], ['--validate-onl'], ['--providers=openai', '--providers=google'], - ['positional'], ['--judge-providers=openai'], ['--providers'], ['--suite=unknown'], - ])('rejects unsafe option input %j', (args) => { + ["--validate-only=false"], + ["--validate-onl"], + ["--providers=openai", "--providers=google"], + ["positional"], + ["--judge-providers=openai"], + ["--providers"], + ["--suite=unknown"], + ])("rejects unsafe option input %j", (args) => { expect(() => parseFixedTraceDiagnosticCliArguments(args)).toThrow(); }); - it('validates a complete bare dry run without credentials, writes, or provider setup', () => { - const output = resolve('/tmp/fixed-trace-diagnostic-cli-no-write.json'); - const result = execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=direct_generation', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, OPENAI_API_KEY: '' } }); - const validated = result.split('\n').map((line) => { - try { return JSON.parse(line) as Record; } catch { return null; } - }).find((line) => line?.diagnosticOnly === true); + it("validates without credentials, writes, provider setup, or dispatch", () => { + const result = execFileSync( + "npx", + [ + "tsx", + "server/tests/manual/fixed-trace-provider-eval.ts", + "--validate-only", + ], + { + cwd: process.cwd(), + encoding: "utf8", + env: { PATH: process.env.PATH ?? "" }, + }, + ); + const lines = result.trim().split("\n"); + expect(lines).toHaveLength(1); + const validated = JSON.parse(lines[0]!) as Record; expect(validated).toMatchObject({ diagnosticOnly: true, - validated: { providers: ['openai'], architectureArm: 'direct_generation', suite: 'canonical', softMaxUsd: 1, outputPath: output }, + dispatchable: false, + outputWritten: false, + providerCalls: 0, }); - expect(existsSync(output)).toBe(false); - }); - - it('binds the reviewed hybrid evaluator suite only to the hybrid arm during validate-only planning', () => { - const output = resolve('/tmp/fixed-trace-diagnostic-cli-hybrid-suite-no-write.json'); - const result = execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=deterministic_policy_llm_fallback_hybrid', '--suite=hybrid-evaluator', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, OPENAI_API_KEY: '' } }); - expect(result).toContain('"suite":"hybrid-evaluator"'); - expect(() => execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', - '--architecture-arm=two_stage_llm_router', '--suite=hybrid-evaluator', '--soft-max-usd=1', `--output=${output}`, - ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); - expect(existsSync(output)).toBe(false); - }); + }, 20_000); - it.each([ - ['--soft-max-usd=0', '--output=/tmp/out.json'], - ['--soft-max-usd=1'], - ])('rejects incomplete dry run configuration', (...args) => { - expect(() => execFileSync('npx', [ - 'tsx', 'server/tests/manual/fixed-trace-provider-eval.ts', '--validate-only', '--providers=openai', ...args, - ], { cwd: process.cwd(), stdio: 'pipe' })).toThrow(); - }); + it.each([["--output=/tmp/out.json"], ["--validate-only=false"]])( + "rejects malformed dry run configuration", + (...args) => { + expect(() => + execFileSync( + "npx", + [ + "tsx", + "server/tests/manual/fixed-trace-provider-eval.ts", + "--validate-only", + ...args, + ], + { cwd: process.cwd(), stdio: "pipe" }, + ), + ).toThrow(); + }, + ); }); diff --git a/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts b/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts index 4a900f3901..6c1a31f341 100644 --- a/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts +++ b/server/tests/unit/addie/fixed-trace-diagnostic-output.test.ts @@ -1,26 +1,26 @@ -import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; import { BudgetedFixedTraceProvider, FixedTraceBudget, fixedTraceResponsePricingPolicy, -} from '../../../src/addie/eval/fixed-trace-budget.js'; +} from "../../../src/addie/eval/fixed-trace-budget.js"; import { assertFixedTraceDiagnosticBudgetReconciliation, runFixedTraceDiagnosticArtifact, type FixedTraceDiagnosticProviderPlan, -} from '../../../src/addie/eval/fixed-trace-diagnostic-run.js'; -import { reserveFixedTraceDiagnosticOutput } from '../../../src/addie/eval/fixed-trace-diagnostic-output.js'; -import { canonicalFixedTraceToolDefinitions } from '../../../src/addie/eval/fixed-trace-tools.js'; -import { fixedTraceHybridPolicy } from '../../../src/addie/eval/fixed-trace-architecture.js'; -import type { FixedTraceProviderStageConfig } from '../../../src/addie/eval/fixed-trace-runner.js'; +} from "../../../src/addie/eval/fixed-trace-diagnostic-run.js"; +import { reserveFixedTraceDiagnosticOutput } from "../../../src/addie/eval/fixed-trace-diagnostic-output.js"; +import { canonicalFixedTraceToolDefinitions } from "../../../src/addie/eval/fixed-trace-tools.js"; +import { fixedTraceHybridPolicy } from "../../../src/addie/eval/fixed-trace-architecture.js"; +import type { FixedTraceProviderStageConfig } from "../../../src/addie/eval/fixed-trace-runner.js"; import { FIXED_TRACE_SUITE, fixedTraceSuiteSha256, type FixedTracePricing, -} from '../../../src/addie/eval/fixed-trace-suite.js'; +} from "../../../src/addie/eval/fixed-trace-suite.js"; import type { ModelProvider, ModelProviderCapabilities, @@ -30,13 +30,13 @@ import type { ModelResponse, NormalizedModelEvent, PreparedModelInvocation, -} from '../../../src/addie/model-providers/model-provider.js'; +} from "../../../src/addie/model-providers/model-provider.js"; const CAPABILITIES: ModelProviderCapabilities = { streaming: false, structuredOutput: true, reasoning: true, - reasoningEfforts: ['none'], + reasoningEfforts: ["none"], customTools: true, providerWebSearch: false, imageInput: false, @@ -44,57 +44,66 @@ const CAPABILITIES: ModelProviderCapabilities = { }; const PRICING: FixedTracePricing = { - profileId: 'anthropic-standard-2026-08:claude-haiku-4-5', + profileId: "anthropic-standard-2026-08:claude-haiku-4-5", inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 5, cacheReadUsdPerMillionTokens: 0.1, cacheWriteUsdPerMillionTokens: 1.25, - cacheReadAccounting: 'additive', - cacheWriteAccounting: 'additive', - source: 'Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.', + cacheReadAccounting: "additive", + cacheWriteAccounting: "additive", + source: + "Repository Anthropic pricing table: Claude Haiku 4.5, refreshed August 2026.", }; -const MODEL = 'claude-haiku-4-5'; -const OPENAI_MODEL = 'gpt-5.6-luna'; +const MODEL = "claude-haiku-4-5"; +const OPENAI_MODEL = "gpt-5.6-luna"; const OPENAI_PRICING: FixedTracePricing = { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', + profileId: "openai-gpt-5.6-luna-2026-08-26", inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset', - cacheWriteAccounting: 'unsupported', - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', + cacheReadAccounting: "subset", + cacheWriteAccounting: "unsupported", + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", }; const ZERO_RATE_PRICING: FixedTracePricing = { ...PRICING, - profileId: 'synthetic-zero-rate-artifact-v1', + profileId: "synthetic-zero-rate-artifact-v1", inputUsdPerMillionTokens: 0, outputUsdPerMillionTokens: 0, - source: 'Synthetic zero-rate artifact pricing.', + source: "Synthetic zero-rate artifact pricing.", }; const DIAGNOSTIC_TEST_REQUEST: ModelRequest = { model: MODEL, system: [], - messages: [{ role: 'user', content: [{ type: 'text', text: 'Synthetic request.' }] }], + messages: [ + { role: "user", content: [{ type: "text", text: "Synthetic request." }] }, + ], tools: [], maxOutputTokens: 1, }; function scriptedRouter( afterFinalResponse?: (response: ModelResponse) => void, - providerId: ModelProviderId = 'anthropic', + providerId: ModelProviderId = "anthropic", ): { provider: ModelProvider; calls: ModelRequest[]; response: ModelResponse } { const calls: ModelRequest[] = []; const response: ModelResponse = { provider: providerId, - model: providerId === 'openai' ? OPENAI_MODEL : MODEL, + model: providerId === "openai" ? OPENAI_MODEL : MODEL, id: `${providerId}-scripted-router-ignore`, - content: [{ type: 'text', text: JSON.stringify({ action: 'ignore', reason: 'Synthetic route.' }) }], - finishReason: 'stop', - providerFinishReason: 'stop', + content: [ + { + type: "text", + text: JSON.stringify({ action: "ignore", reason: "Synthetic route." }), + }, + ], + finishReason: "stop", + providerFinishReason: "stop", usage: { inputTokens: 10, outputTokens: 5 }, }; const provider: ModelProvider = { @@ -106,15 +115,30 @@ function scriptedRouter( model: request.model, capabilities: CAPABILITIES, requestMetadata: request.requestMetadata, - providerRequest: structuredClone(request) as unknown as Readonly>, + providerRequest: structuredClone(request) as unknown as Readonly< + Record + >, }; }, - async *respond(request: ModelRequest, options: ModelRespondOptions = {}): AsyncIterable { + async *respond( + request: ModelRequest, + options: ModelRespondOptions = {}, + ): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); calls.push(structuredClone(request)); - yield { type: 'response_start', provider: providerId, model: response.model, id: response.id }; - yield { type: 'text_delta', index: 0, text: response.content[0].type === 'text' ? response.content[0].text : '' }; - yield { type: 'response_complete', response }; + yield { + type: "response_start", + provider: providerId, + model: response.model, + id: response.id, + }; + yield { + type: "text_delta", + index: 0, + text: + response.content[0].type === "text" ? response.content[0].text : "", + }; + yield { type: "response_complete", response }; afterFinalResponse?.(response); }, }; @@ -131,29 +155,55 @@ function cloneChangingIdentityProvider(): { const provider: ModelProvider = { get id(): ModelProviderId { reads++; - return reads === 1 ? 'anthropic' : 'openai'; + return reads === 1 ? "anthropic" : "openai"; }, capabilities: CAPABILITIES, prepare(request): PreparedModelInvocation { // The delegate's request surface is stable; only an old clone's second // read of `id` would change the wrapper identity. return { - provider: 'anthropic', model: request.model, capabilities: CAPABILITIES, + provider: "anthropic", + model: request.model, + capabilities: CAPABILITIES, requestMetadata: request.requestMetadata, - providerRequest: structuredClone(request) as unknown as Readonly>, + providerRequest: structuredClone(request) as unknown as Readonly< + Record + >, }; }, async *respond(request, options = {}): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); calls.push(structuredClone(request)); const response: ModelResponse = { - provider: 'anthropic', model: MODEL, id: 'stable-response', - content: [{ type: 'text', text: JSON.stringify({ action: 'ignore', reason: 'Synthetic route.' }) }], - finishReason: 'stop', providerFinishReason: 'stop', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "stable-response", + content: [ + { + type: "text", + text: JSON.stringify({ + action: "ignore", + reason: "Synthetic route.", + }), + }, + ], + finishReason: "stop", + providerFinishReason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, + }; + yield { + type: "response_start", + provider: "anthropic", + model: response.model, + id: response.id, + }; + yield { + type: "text_delta", + index: 0, + text: + response.content[0].type === "text" ? response.content[0].text : "", }; - yield { type: 'response_start', provider: 'anthropic', model: response.model, id: response.id }; - yield { type: 'text_delta', index: 0, text: response.content[0].type === 'text' ? response.content[0].text : '' }; - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; }, }; return { provider, calls, idReads: () => reads }; @@ -161,17 +211,19 @@ function cloneChangingIdentityProvider(): { function stage( provider: ModelProvider, - pricing: FixedTracePricing = provider.id === 'openai' ? OPENAI_PRICING : PRICING, + pricing: FixedTracePricing = provider.id === "openai" + ? OPENAI_PRICING + : PRICING, ): FixedTraceProviderStageConfig { return { provider, - model: provider.id === 'openai' ? OPENAI_MODEL : MODEL, - reasoningEffort: 'none', + model: provider.id === "openai" ? OPENAI_MODEL : MODEL, + reasoningEffort: "none", maxOutputTokens: 300, timeoutMs: 30_000, maxIterations: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', + samplingMode: "provider_no_sampling_control", temperature: null, pricing: structuredClone(pricing), }; @@ -180,7 +232,9 @@ function stage( function budgetedStage( provider: ModelProvider, budget: FixedTraceBudget, - pricing: FixedTracePricing = provider.id === 'openai' ? OPENAI_PRICING : PRICING, + pricing: FixedTracePricing = provider.id === "openai" + ? OPENAI_PRICING + : PRICING, ): FixedTraceProviderStageConfig { const configured = stage(provider, pricing); return { @@ -189,92 +243,156 @@ function budgetedStage( provider, budget, configured.pricing, - fixedTraceResponsePricingPolicy(provider.id, configured.model, configured.pricing), + fixedTraceResponsePricingPolicy( + provider.id, + configured.model, + configured.pricing, + ), ), }; } -function twoTurnProvider( - afterRouterResponse?: () => void, -): { provider: ModelProvider; calls: ModelRequest[] } { +function twoTurnProvider(afterRouterResponse?: () => void): { + provider: ModelProvider; + calls: ModelRequest[]; +} { const calls: ModelRequest[] = []; let generationTurn = 0; const provider: ModelProvider = { - id: 'anthropic', + id: "anthropic", capabilities: CAPABILITIES, prepare(request): PreparedModelInvocation { return { - provider: 'anthropic', model: request.model, capabilities: CAPABILITIES, + provider: "anthropic", + model: request.model, + capabilities: CAPABILITIES, requestMetadata: request.requestMetadata, - providerRequest: structuredClone(request) as unknown as Readonly>, + providerRequest: structuredClone(request) as unknown as Readonly< + Record + >, }; }, async *respond(request, options = {}): AsyncIterable { await options.beforeDispatch?.(this.prepare(request)); calls.push(structuredClone(request)); - const router = request.requestMetadata?.purpose === 'fixed_trace_router'; + const router = request.requestMetadata?.purpose === "fixed_trace_router"; const response: ModelResponse = router ? { - provider: 'anthropic', model: MODEL, id: 'router', - content: [{ type: 'text', text: JSON.stringify({ - action: 'respond', tool_sets: ['knowledge'], confidence: 'high', - requires_depth: false, reason: 'Synthetic route.', - }) }], - finishReason: 'stop', providerFinishReason: 'stop', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "router", + content: [ + { + type: "text", + text: JSON.stringify({ + action: "respond", + tool_sets: ["knowledge"], + confidence: "high", + requires_depth: false, + reason: "Synthetic route.", + }), + }, + ], + finishReason: "stop", + providerFinishReason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, } : generationTurn++ === 0 ? { - provider: 'anthropic', model: MODEL, id: 'generation-tool', - content: [{ type: 'tool_call', id: 'tool-1', name: 'search_docs', input: { query: 'task model' } }], - finishReason: 'tool_calls', providerFinishReason: 'tool_use', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "generation-tool", + content: [ + { + type: "tool_call", + id: "tool-1", + name: "search_docs", + input: { query: "task model" }, + }, + ], + finishReason: "tool_calls", + providerFinishReason: "tool_use", + usage: { inputTokens: 10, outputTokens: 5 }, } : { - provider: 'anthropic', model: MODEL, id: 'generation-final', - content: [{ type: 'text', text: 'A buyer calls a seller task and receives its structured response.' }], - finishReason: 'stop', providerFinishReason: 'stop', usage: { inputTokens: 10, outputTokens: 5 }, + provider: "anthropic", + model: MODEL, + id: "generation-final", + content: [ + { + type: "text", + text: "A buyer calls a seller task and receives its structured response.", + }, + ], + finishReason: "stop", + providerFinishReason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, }; - yield { type: 'response_start', provider: 'anthropic', model: response.model, id: response.id }; + yield { + type: "response_start", + provider: "anthropic", + model: response.model, + id: response.id, + }; for (const [index, content] of response.content.entries()) { - if (content.type === 'text') yield { type: 'text_delta', index, text: content.text }; - if (content.type === 'tool_call') yield { type: 'tool_call', index, call: content }; + if (content.type === "text") + yield { type: "text_delta", index, text: content.text }; + if (content.type === "tool_call") + yield { type: "tool_call", index, call: content }; } - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; if (router) afterRouterResponse?.(); }, }; return { provider, calls }; } -describe('fixed-trace diagnostic output reservation', () => { - it('never overwrites an existing artifact', () => { - const path = join(mkdtempSync(join(tmpdir(), 'fixed-trace-output-')), 'artifact.json'); - writeFileSync(path, 'existing'); - expect(() => reserveFixedTraceDiagnosticOutput(path)).toThrow('Cannot exclusively reserve'); - expect(readFileSync(path, 'utf8')).toBe('existing'); +describe("fixed-trace diagnostic output reservation", () => { + it("never overwrites an existing artifact", () => { + const path = join( + mkdtempSync(join(tmpdir(), "fixed-trace-output-")), + "artifact.json", + ); + writeFileSync(path, "existing"); + expect(() => reserveFixedTraceDiagnosticOutput(path)).toThrow( + "Cannot exclusively reserve", + ); + expect(readFileSync(path, "utf8")).toBe("existing"); }); - it('rejects directory and missing-parent targets before dispatch', () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - expect(() => reserveFixedTraceDiagnosticOutput(directory)).toThrow('Cannot exclusively reserve'); - expect(() => reserveFixedTraceDiagnosticOutput(join(directory, 'missing', 'artifact.json'))).toThrow('Cannot exclusively reserve'); + it("rejects directory and missing-parent targets before dispatch", () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + expect(() => reserveFixedTraceDiagnosticOutput(directory)).toThrow( + "Cannot exclusively reserve", + ); + expect(() => + reserveFixedTraceDiagnosticOutput( + join(directory, "missing", "artifact.json"), + ), + ).toThrow("Cannot exclusively reserve"); }); - it('claims then finalizes through one exclusive descriptor', () => { - const path = join(mkdtempSync(join(tmpdir(), 'fixed-trace-output-')), 'artifact.json'); + it("claims then finalizes through one exclusive descriptor", () => { + const path = join( + mkdtempSync(join(tmpdir(), "fixed-trace-output-")), + "artifact.json", + ); const reservation = reserveFixedTraceDiagnosticOutput(path); reservation.finalize('{"diagnosticOnly":true}\n'); - expect(readFileSync(path, 'utf8')).toBe('{"diagnosticOnly":true}\n'); + expect(readFileSync(path, "utf8")).toBe('{"diagnosticOnly":true}\n'); }); - it('runs the manual diagnostic candidate path into a complete reserved artifact with scripted providers', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("runs the manual diagnostic candidate path into a complete reserved artifact with scripted providers", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const plan: FixedTraceDiagnosticProviderPlan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; @@ -282,26 +400,26 @@ describe('fixed-trace diagnostic output reservation', () => { const artifact = await runFixedTraceDiagnosticArtifact({ plans: [plan], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), - gitCommit: 'abcdef0', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', + promptConfigVersion: "synthetic-manual-prompt-v1", traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', - architectureArm: 'deterministic_policy_llm_fallback_hybrid', + toolDefinitionProvenance: "fixture_local", + architectureArm: "deterministic_policy_llm_fallback_hybrid", hybridPolicy: fixedTraceHybridPolicy(), }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); - const persisted = JSON.parse(readFileSync(path, 'utf8')) as typeof artifact; + const persisted = JSON.parse(readFileSync(path, "utf8")) as typeof artifact; expect(router.calls).toHaveLength(1); expect(persisted).toMatchObject({ complete: true, @@ -309,19 +427,28 @@ describe('fixed-trace diagnostic output reservation', () => { comparisonEligible: false, promotionEvidenceEligible: false, rolloutPass: false, - architectureArm: { id: 'deterministic_policy_llm_fallback_hybrid', diagnosticOnly: true }, + architectureArm: { + id: "deterministic_policy_llm_fallback_hybrid", + diagnosticOnly: true, + }, hybridPolicy: fixedTraceHybridPolicy(), - runs: [{ - provider: 'anthropic', - summary: { complete: true, comparisonEligible: false }, - observations: [{ traceId: selectedTrace.id, terminalStatus: 'ignored' }], - }], + runs: [ + { + provider: "anthropic", + summary: { complete: true, comparisonEligible: false }, + observations: [ + { traceId: selectedTrace.id, terminalStatus: "ignored" }, + ], + }, + ], }); expect(persisted.runs[0].observations).toHaveLength(1); - expect(artifact.runs[0].runId).toBe('synthetic-manual-root:anthropic'); - expect(artifact.runs[0].observations.every((observation) => ( - observation.metadata.runId === artifact.runs[0].runId - ))).toBe(true); + expect(artifact.runs[0].runId).toBe("synthetic-manual-root:anthropic"); + expect( + artifact.runs[0].observations.every( + (observation) => observation.metadata.runId === artifact.runs[0].runId, + ), + ).toBe(true); expect(artifact.budget).toMatchObject({ accountedSpendUsd: 0.000035, dispatchedCalls: 1, @@ -335,40 +462,44 @@ describe('fixed-trace diagnostic output reservation', () => { estimatedCostUsd: 0.000035, }); expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBe(0.000035); - expect(persisted.runs[0].observations[0].metadata.router.usage).toMatchObject({ inputTokens: 10 }); + expect( + persisted.runs[0].observations[0].metadata.router.usage, + ).toMatchObject({ inputTokens: 10 }); }); - it('freezes the complete two-plan artifact contract before a provider can mutate later plans', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); - const sourceBundleFiles = ['before.ts']; + it("freezes the complete two-plan artifact contract before a provider can mutate later plans", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); + const sourceBundleFiles = ["before.ts"]; const budget = new FixedTraceBudget(1); const baseConfig = { - sourceBundleSha256: 'a'.repeat(64), - gitCommit: 'abcdef0', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", gitDirty: false, - promptConfigVersion: 'before-prompt', + promptConfigVersion: "before-prompt", traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local' as const, - architectureArm: 'two_stage_llm_router' as const, + toolDefinitionProvenance: "fixture_local" as const, + architectureArm: "two_stage_llm_router" as const, }; let secondPlan!: FixedTraceDiagnosticProviderPlan; const first = scriptedRouter(() => { - baseConfig.sourceBundleSha256 = 'b'.repeat(64); - baseConfig.promptConfigVersion = 'forged-after-first-plan'; - sourceBundleFiles.push('forged-after-first-plan.ts'); + baseConfig.sourceBundleSha256 = "b".repeat(64); + baseConfig.promptConfigVersion = "forged-after-first-plan"; + sourceBundleFiles.push("forged-after-first-plan.ts"); secondPlan.router.maxOutputTokens = 1; - secondPlan.router.pricing.source = 'forged-after-first-plan'; + secondPlan.router.pricing.source = "forged-after-first-plan"; }); const second = scriptedRouter(() => { first.response.usage.inputTokens = 999_999; - }, 'openai'); + }, "openai"); secondPlan = { - name: 'openai', + name: "openai", router: budgetedStage(second.provider, budget), generation: budgetedStage(second.provider, budget), }; @@ -376,7 +507,7 @@ describe('fixed-trace diagnostic output reservation', () => { const artifact = await runFixedTraceDiagnosticArtifact({ plans: [ { - name: 'anthropic', + name: "anthropic", router: budgetedStage(first.provider, budget), generation: budgetedStage(first.provider, budget), }, @@ -385,27 +516,30 @@ describe('fixed-trace diagnostic output reservation', () => { baseConfig, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", sourceBundleFiles, - budgetNote: 'Synthetic no-network budget note.', + budgetNote: "Synthetic no-network budget note.", }); - const persisted = JSON.parse(readFileSync(path, 'utf8')) as typeof artifact; + const persisted = JSON.parse(readFileSync(path, "utf8")) as typeof artifact; expect(persisted).toMatchObject({ - sourceBundleSha256: 'a'.repeat(64), - promptConfigVersion: 'before-prompt', - sourceBundleFiles: ['before.ts'], - requestedProviders: ['anthropic', 'openai'], + sourceBundleSha256: "a".repeat(64), + promptConfigVersion: "before-prompt", + sourceBundleFiles: ["before.ts"], + requestedProviders: ["anthropic", "openai"], }); expect(artifact.runs.map((run) => run.runId)).toEqual([ - 'synthetic-manual-root:anthropic', - 'synthetic-manual-root:openai', + "synthetic-manual-root:anthropic", + "synthetic-manual-root:openai", ]); expect(persisted.runs[1].requestedConfig.router).toMatchObject({ - provider: 'openai', + provider: "openai", maxOutputTokens: 300, - pricing: { source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.' }, + pricing: { + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", + }, }); expect(persisted.runs[0].observations[0].metadata.router).toMatchObject({ usage: { inputTokens: 10, outputTokens: 5 }, @@ -422,58 +556,71 @@ describe('fixed-trace diagnostic output reservation', () => { } }); - it('rejects a provider-mismatched plan before scripted dispatch', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a provider-mismatched plan before scripted dispatch", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const baseConfig = { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local' as const, - architectureArm: 'two_stage_llm_router' as const, + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local" as const, + architectureArm: "two_stage_llm_router" as const, }; - const invoke = (plans: FixedTraceDiagnosticProviderPlan[]) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', - }); + const invoke = (plans: FixedTraceDiagnosticProviderPlan[]) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); const plan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; - await expect(invoke([{ ...plan, name: 'not-anthropic' }])).rejects.toThrow('provider plans require unique names'); - const duplicatePath = join(directory, 'duplicate-artifact.json'); - await expect(runFixedTraceDiagnosticArtifact({ - plans: [plan, { ...plan }], - baseConfig, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(duplicatePath), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', - })).rejects.toThrow('provider plans require unique names'); + await expect(invoke([{ ...plan, name: "not-anthropic" }])).rejects.toThrow( + "provider plans require unique names", + ); + const duplicatePath = join(directory, "duplicate-artifact.json"); + await expect( + runFixedTraceDiagnosticArtifact({ + plans: [plan, { ...plan }], + baseConfig, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(duplicatePath), + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }), + ).rejects.toThrow("provider plans require unique names"); expect(router.calls).toHaveLength(0); - expect(readFileSync(path, 'utf8')).toBe(''); - expect(readFileSync(duplicatePath, 'utf8')).toBe(''); + expect(readFileSync(path, "utf8")).toBe(""); + expect(readFileSync(duplicatePath, "utf8")).toBe(""); }); - it('rejects a plan identity accessor before it can change validation into execution', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'failed-artifact.json'); - const completedPath = join(directory, 'completed-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a plan identity accessor before it can change validation into execution", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "failed-artifact.json"); + const completedPath = join(directory, "completed-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); let nameReads = 0; @@ -483,176 +630,308 @@ describe('fixed-trace diagnostic output reservation', () => { // before either a lease or a provider dispatch is possible. get name() { nameReads++; - return nameReads <= 6 ? 'anthropic' : 'forged'; + return nameReads <= 6 ? "anthropic" : "forged"; }, router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; - const invoke = (plans: readonly FixedTraceDiagnosticProviderPlan[], path: string) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - }); + const invoke = ( + plans: readonly FixedTraceDiagnosticProviderPlan[], + path: string, + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([accessorPlan] as unknown as FixedTraceDiagnosticProviderPlan[], failedPath)) - .rejects.toThrow('provider plan 0.name must be an own data property'); + await expect( + invoke( + [accessorPlan] as unknown as FixedTraceDiagnosticProviderPlan[], + failedPath, + ), + ).rejects.toThrow("provider plan 0.name must be an own data property"); expect(nameReads).toBe(0); expect(router.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); + expect(readFileSync(failedPath, "utf8")).toBe(""); expect(budget.snapshot()).toMatchObject({ - accountedSpendUsd: 0, reservedUsd: 0, dispatchedCalls: 0, - completedCalls: 0, budgetRejectedCalls: 0, admissionClosed: false, exposureUnknown: false, + accountedSpendUsd: 0, + reservedUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 0, + admissionClosed: false, + exposureUnknown: false, }); - const artifact = await invoke([{ - name: 'anthropic', router: accessorPlan.router, generation: accessorPlan.generation, - }], completedPath); + const artifact = await invoke( + [ + { + name: "anthropic", + router: accessorPlan.router, + generation: accessorPlan.generation, + }, + ], + completedPath, + ); expect(router.calls).toHaveLength(1); - expect(artifact.runs[0]).toMatchObject({ provider: 'anthropic', runId: 'root:anthropic' }); + expect(artifact.runs[0]).toMatchObject({ + provider: "anthropic", + runId: "root:anthropic", + }); }); - it('never rereads a delegate identity while cloning an authenticated plan', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("never rereads a delegate identity while cloning an authenticated plan", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const changing = cloneChangingIdentityProvider(); const budget = new FixedTraceBudget(1); const wrapper = new BudgetedFixedTraceProvider( changing.provider, budget, PRICING, - fixedTraceResponsePricingPolicy('anthropic', MODEL, PRICING), + fixedTraceResponsePricingPolicy("anthropic", MODEL, PRICING), ); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: stage(wrapper), generation: stage(wrapper) }], + plans: [ + { + name: "anthropic", + router: stage(wrapper), + generation: stage(wrapper), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(changing.idReads()).toBe(1); expect(changing.calls).toHaveLength(1); - expect(artifact).toMatchObject({ requestedProviders: ['anthropic'] }); - expect(artifact.runs[0]).toMatchObject({ provider: 'anthropic', runId: 'root:anthropic' }); + expect(artifact).toMatchObject({ requestedProviders: ["anthropic"] }); + expect(artifact.runs[0]).toMatchObject({ + provider: "anthropic", + runId: "root:anthropic", + }); expect(artifact.runs[0].observations[0].metadata.router).toMatchObject({ - requestedProvider: 'anthropic', returnedProvider: 'anthropic', estimatedCostUsd: 0.000035, + requestedProvider: "anthropic", + returnedProvider: "anthropic", + estimatedCostUsd: 0.000035, }); }); - it('rejects a self-declared zero-rate profile before lease or dispatch and leaves the budget reusable', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'forged-artifact.json'); - const retryPath = join(directory, 'retry-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a self-declared zero-rate profile before lease or dispatch and leaves the budget reusable", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "forged-artifact.json"); + const retryPath = join(directory, "retry-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1e-12); const trustedRouter = budgetedStage(router.provider, budget); const trustedGeneration = budgetedStage(router.provider, budget); const forgedPricing: FixedTracePricing = { ...ZERO_RATE_PRICING, - profileId: 'attacker-says-reviewed-v1', - source: 'attacker assertion', + profileId: "attacker-says-reviewed-v1", + source: "attacker assertion", }; - const invoke = (plans: readonly FixedTraceDiagnosticProviderPlan[], path: string) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - }); + const invoke = ( + plans: readonly FixedTraceDiagnosticProviderPlan[], + path: string, + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([{ - name: 'anthropic', router: { ...trustedRouter, pricing: forgedPricing }, generation: trustedGeneration, - }], failedPath)).rejects.toThrow('Fixed trace pricing profile is not evaluator approved'); + await expect( + invoke( + [ + { + name: "anthropic", + router: { ...trustedRouter, pricing: forgedPricing }, + generation: trustedGeneration, + }, + ], + failedPath, + ), + ).rejects.toThrow("Fixed trace pricing profile is not evaluator approved"); expect(router.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); + expect(readFileSync(failedPath, "utf8")).toBe(""); expect(budget.snapshot()).toMatchObject({ - accountedSpendUsd: 0, reservedUsd: 0, dispatchedCalls: 0, - completedCalls: 0, budgetRejectedCalls: 0, admissionClosed: false, exposureUnknown: false, + accountedSpendUsd: 0, + reservedUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 0, + admissionClosed: false, + exposureUnknown: false, }); - const artifact = await invoke([{ - name: 'anthropic', router: trustedRouter, generation: trustedGeneration, - }], retryPath); - expect(artifact.budget).toMatchObject({ dispatchedCalls: 0, completedCalls: 0, budgetRejectedCalls: 1 }); - expect(readFileSync(retryPath, 'utf8')).not.toBe(''); + const artifact = await invoke( + [ + { + name: "anthropic", + router: trustedRouter, + generation: trustedGeneration, + }, + ], + retryPath, + ); + expect(artifact.budget).toMatchObject({ + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 1, + }); + expect(readFileSync(retryPath, "utf8")).not.toBe(""); }); - it('rejects nested pricing accessors without reading them and leaves the budget reusable', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'accessor-artifact.json'); - const retryPath = join(directory, 'retry-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects nested pricing accessors without reading them and leaves the budget reusable", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "accessor-artifact.json"); + const retryPath = join(directory, "retry-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const trustedRouter = budgetedStage(router.provider, budget); const trustedGeneration = budgetedStage(router.provider, budget); const accessorPricing = { ...PRICING } as FixedTracePricing; let pricingReads = 0; - Object.defineProperty(accessorPricing, 'inputUsdPerMillionTokens', { + Object.defineProperty(accessorPricing, "inputUsdPerMillionTokens", { enumerable: true, - get() { pricingReads++; return 0; }, - }); - const invoke = (plans: readonly FixedTraceDiagnosticProviderPlan[], path: string) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + get() { + pricingReads++; + return 0; }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', }); + const invoke = ( + plans: readonly FixedTraceDiagnosticProviderPlan[], + path: string, + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([{ - name: 'anthropic', router: { ...trustedRouter, pricing: accessorPricing }, generation: trustedGeneration, - }], failedPath)).rejects.toThrow('provider plan 0.router.pricing.inputUsdPerMillionTokens must be an own data property'); + await expect( + invoke( + [ + { + name: "anthropic", + router: { ...trustedRouter, pricing: accessorPricing }, + generation: trustedGeneration, + }, + ], + failedPath, + ), + ).rejects.toThrow( + "provider plan 0.router.pricing.inputUsdPerMillionTokens must be an own data property", + ); expect(pricingReads).toBe(0); expect(router.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); - expect(budget.snapshot()).toMatchObject({ dispatchedCalls: 0, completedCalls: 0, budgetRejectedCalls: 0 }); + expect(readFileSync(failedPath, "utf8")).toBe(""); + expect(budget.snapshot()).toMatchObject({ + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 0, + }); - await invoke([{ name: 'anthropic', router: trustedRouter, generation: trustedGeneration }], retryPath); + await invoke( + [ + { + name: "anthropic", + router: trustedRouter, + generation: trustedGeneration, + }, + ], + retryPath, + ); expect(router.calls).toHaveLength(1); }); - it('does not let a final-response mutation alter its snapshotted manual artifact', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("does not let a final-response mutation alter its snapshotted manual artifact", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); let plan!: FixedTraceDiagnosticProviderPlan; - const router = scriptedRouter(() => { plan.router.model = 'mutated-after-final-response'; }); + const router = scriptedRouter(() => { + plan.router.model = "mutated-after-final-response"; + }); const budget = new FixedTraceBudget(1); plan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget), }; @@ -660,62 +939,84 @@ describe('fixed-trace diagnostic output reservation', () => { const artifact = await runFixedTraceDiagnosticArtifact({ plans: [plan], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), - gitCommit: 'abcdef0', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', + promptConfigVersion: "synthetic-manual-prompt-v1", traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', - architectureArm: 'two_stage_llm_router', + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'synthetic-manual-root', - runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], - budgetNote: 'Synthetic no-network budget note.', + runRootId: "synthetic-manual-root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(router.calls).toHaveLength(1); expect(artifact.runs[0].requestedConfig.router.model).toBe(MODEL); - expect(JSON.parse(readFileSync(path, 'utf8'))).toMatchObject({ complete: true, diagnosticOnly: true }); + expect(JSON.parse(readFileSync(path, "utf8"))).toMatchObject({ + complete: true, + diagnosticOnly: true, + }); }); - it('derives child run IDs internally instead of accepting an unrelated callback result', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("derives child run IDs internally instead of accepting an unrelated callback result", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(1); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget) }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, // This former input is intentionally ignored at runtime as well as // removed from the public type, so a JavaScript caller cannot forge it. - runIdForProvider: () => 'unrelated-id', + runIdForProvider: () => "unrelated-id", budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", } as unknown as Parameters[0]); - expect(artifact.runs[0].runId).toBe('root:anthropic'); - expect(artifact.runs[0].observations[0].metadata.runId).toBe('root:anthropic'); + expect(artifact.runs[0].runId).toBe("root:anthropic"); + expect(artifact.runs[0].observations[0].metadata.runId).toBe( + "root:anthropic", + ); }); - it('rejects a subclass that claims budget binding while bypassing the ledger', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a subclass that claims budget binding while bypassing the ledger", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const delegate = scriptedRouter(); const budget = new FixedTraceBudget(1); class BypassingBudgetProvider extends BudgetedFixedTraceProvider { @@ -724,13 +1025,15 @@ describe('fixed-trace diagnostic output reservation', () => { delegate.provider, budget, PRICING, - fixedTraceResponsePricingPolicy('anthropic', MODEL, PRICING), + fixedTraceResponsePricingPolicy("anthropic", MODEL, PRICING), ); } // This was previously trusted through instanceof plus a public, // overridable isBoundToBudget predicate. - isBoundToBudget(): boolean { return true; } + isBoundToBudget(): boolean { + return true; + } override async *respond( request: ModelRequest, @@ -741,51 +1044,85 @@ describe('fixed-trace diagnostic output reservation', () => { } const bypass = new BypassingBudgetProvider(); - await expect(runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: stage(bypass), generation: stage(bypass) }], - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - })).rejects.toThrow('provider plans require unique names'); + await expect( + runFixedTraceDiagnosticArtifact({ + plans: [ + { + name: "anthropic", + router: stage(bypass), + generation: stage(bypass), + }, + ], + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }), + ).rejects.toThrow("provider plans require unique names"); expect(delegate.calls).toHaveLength(0); - expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0, dispatchedCalls: 0, completedCalls: 0 }); - expect(readFileSync(path, 'utf8')).toBe(''); + expect(budget.snapshot()).toMatchObject({ + accountedSpendUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + }); + expect(readFileSync(path, "utf8")).toBe(""); }); - it('keeps collector, metadata, summary, ledger, and artifact on the terminal snapshot', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("keeps collector, metadata, summary, ledger, and artifact on the terminal snapshot", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter((response) => { - response.id = 'forged-id'; - response.model = 'forged-model'; - response.content[0] = { type: 'text', text: 'forged response' }; + response.id = "forged-id"; + response.model = "forged-model"; + response.content[0] = { type: "text", text: "forged response" }; response.usage.inputTokens = 999_999; response.usage.outputTokens = 999_999; }); const budget = new FixedTraceBudget(1); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget) }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); - const persisted = JSON.parse(readFileSync(path, 'utf8')) as typeof artifact; + const persisted = JSON.parse(readFileSync(path, "utf8")) as typeof artifact; const observation = artifact.runs[0].observations[0]; expect(observation.metadata.router).toMatchObject({ @@ -794,111 +1131,183 @@ describe('fixed-trace diagnostic output reservation', () => { estimatedCostUsd: 0.000035, }); expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBe(0.000035); - expect(artifact.budget).toMatchObject({ accountedSpendUsd: 0.000035, dispatchedCalls: 1, completedCalls: 1 }); - expect(persisted.runs[0].observations[0].metadata.router.usage).toMatchObject({ inputTokens: 10, outputTokens: 5 }); + expect(artifact.budget).toMatchObject({ + accountedSpendUsd: 0.000035, + dispatchedCalls: 1, + completedCalls: 1, + }); + expect( + persisted.runs[0].observations[0].metadata.router.usage, + ).toMatchObject({ inputTokens: 10, outputTokens: 5 }); expect(persisted.budget.accountedSpendUsd).toBe(0.000035); }); - it('retains an unknown-model response as unknown exposure without inventing a spend equality', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("retains an unknown-model response as unknown exposure without inventing a spend equality", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); - router.response.model = 'unapproved-model'; + router.response.model = "unapproved-model"; const budget = new FixedTraceBudget(1); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ - name: 'anthropic', - router: budgetedStage(router.provider, budget), - generation: budgetedStage(router.provider, budget), - }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(artifact.runs[0].observations[0].metadata.router).toMatchObject({ - source: 'provider', returnedModel: 'unapproved-model', estimatedCostUsd: null, + source: "provider", + returnedModel: "unapproved-model", + estimatedCostUsd: null, }); expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBeNull(); expect(artifact.budget).toMatchObject({ - accountedSpendUsd: 0, dispatchedCalls: 1, completedCalls: 0, exposureUnknown: true, + accountedSpendUsd: 0, + dispatchedCalls: 1, + completedCalls: 0, + exposureUnknown: true, }); }); - it('rejects a settled ledger for an unpriced dispatched provider response', () => { + it("rejects a settled ledger for an unpriced dispatched provider response", () => { const providerStage = { - source: 'provider', dispatched: true, dispatchedCalls: 1, - usageKnown: true, usage: { inputTokens: 10, outputTokens: 5 }, estimatedCostUsd: null, + source: "provider", + dispatched: true, + dispatchedCalls: 1, + usageKnown: true, + usage: { inputTokens: 10, outputTokens: 5 }, + estimatedCostUsd: null, }; const notRunStage = { - source: 'not_run', dispatched: false, dispatchedCalls: 0, - usageKnown: false, usage: null, estimatedCostUsd: 0, + source: "not_run", + dispatched: false, + dispatchedCalls: 0, + usageKnown: false, + usage: null, + estimatedCostUsd: 0, }; - expect(() => assertFixedTraceDiagnosticBudgetReconciliation({ - policy: 'soft_admission_target', softMaxUsd: 1, accountedSpendUsd: 0.000035, - reservedUsd: 0, remainingUsd: 0.999965, dispatchedCalls: 1, completedCalls: 1, - budgetRejectedCalls: 0, admissionClosed: false, exposureUnknown: false, - }, [{ observations: [{ - terminalStatus: 'complete', - metadata: { router: providerStage, generation: notRunStage }, - }] }] as never)).toThrow('unpriced dispatched response lacks unknown budget exposure'); + expect(() => + assertFixedTraceDiagnosticBudgetReconciliation( + { + policy: "soft_admission_target", + softMaxUsd: 1, + accountedSpendUsd: 0.000035, + reservedUsd: 0, + remainingUsd: 0.999965, + dispatchedCalls: 1, + completedCalls: 1, + budgetRejectedCalls: 0, + admissionClosed: false, + exposureUnknown: false, + }, + [ + { + observations: [ + { + terminalStatus: "complete", + metadata: { router: providerStage, generation: notRunStage }, + }, + ], + }, + ] as never, + ), + ).toThrow("unpriced dispatched response lacks unknown budget exposure"); }); - it('reconciles a pre-dispatch budget rejection with a local/not-run observation', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("reconciles a pre-dispatch budget rejection with a local/not-run observation", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const router = scriptedRouter(); const budget = new FixedTraceBudget(0.000001); const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(router.provider, budget), generation: budgetedStage(router.provider, budget) }], + plans: [ + { + name: "anthropic", + router: budgetedStage(router.provider, budget), + generation: budgetedStage(router.provider, budget), + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); expect(artifact.runs[0].observations[0]).toMatchObject({ - terminalStatus: 'not_dispatched_budget', - metadata: { router: { source: 'local', dispatched: false }, generation: { source: 'not_run' } }, + terminalStatus: "not_dispatched_budget", + metadata: { + router: { source: "local", dispatched: false }, + generation: { source: "not_run" }, + }, }); expect(artifact.budget).toMatchObject({ - accountedSpendUsd: 0, dispatchedCalls: 0, completedCalls: 0, budgetRejectedCalls: 1, exposureUnknown: false, + accountedSpendUsd: 0, + dispatchedCalls: 0, + completedCalls: 0, + budgetRejectedCalls: 1, + exposureUnknown: false, }); }); - it('preflights every plan before leasing a pristine budget or dispatching an earlier plan', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const failedPath = join(directory, 'failed-artifact.json'); - const completedPath = join(directory, 'completed-artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); - const first = scriptedRouter(undefined, 'anthropic'); - const second = scriptedRouter(undefined, 'openai'); + it("preflights every plan before leasing a pristine budget or dispatching an earlier plan", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const failedPath = join(directory, "failed-artifact.json"); + const completedPath = join(directory, "completed-artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); + const first = scriptedRouter(undefined, "anthropic"); + const second = scriptedRouter(undefined, "openai"); const budget = new FixedTraceBudget(1); const firstPlan: FixedTraceDiagnosticProviderPlan = { - name: 'anthropic', + name: "anthropic", router: budgetedStage(first.provider, budget), generation: budgetedStage(first.provider, budget), }; const invalidSecondPlan: FixedTraceDiagnosticProviderPlan = { - name: 'openai', + name: "openai", router: budgetedStage(second.provider, budget), generation: budgetedStage(second.provider, budget), }; @@ -906,26 +1315,34 @@ describe('fixed-trace diagnostic output reservation', () => { const invoke = ( plans: readonly FixedTraceDiagnosticProviderPlan[], path: string, - ) => runFixedTraceDiagnosticArtifact({ - plans, - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - }); + ) => + runFixedTraceDiagnosticArtifact({ + plans, + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput(path), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }); - await expect(invoke([firstPlan, invalidSecondPlan], failedPath)).rejects.toThrow( - 'generation maxIterations must be between', - ); + await expect( + invoke([firstPlan, invalidSecondPlan], failedPath), + ).rejects.toThrow("generation maxIterations must be between"); expect(first.calls).toHaveLength(0); expect(second.calls).toHaveLength(0); - expect(readFileSync(failedPath, 'utf8')).toBe(''); + expect(readFileSync(failedPath, "utf8")).toBe(""); expect(budget.snapshot()).toMatchObject({ accountedSpendUsd: 0, reservedUsd: 0, @@ -937,7 +1354,7 @@ describe('fixed-trace diagnostic output reservation', () => { }); const validSecondPlan: FixedTraceDiagnosticProviderPlan = { - name: 'openai', + name: "openai", router: budgetedStage(second.provider, budget), generation: budgetedStage(second.provider, budget), }; @@ -953,93 +1370,177 @@ describe('fixed-trace diagnostic output reservation', () => { expect(artifact.budget.accountedSpendUsd).toBeCloseTo(0.000043); }); - it('prevents post-preflight method and prototype tampering in a two-turn zero-rate run', async () => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); - const path = join(directory, 'artifact.json'); - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'knowledge-task-model'); - if (!selectedTrace) throw new Error('Missing synthetic knowledge trace'); + it("prevents post-preflight method and prototype tampering in a two-turn zero-rate run", async () => { + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); + const path = join(directory, "artifact.json"); + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "knowledge-task-model", + ); + if (!selectedTrace) throw new Error("Missing synthetic knowledge trace"); const attacks: string[] = []; let generation!: BudgetedFixedTraceProvider; const delegate = twoTurnProvider(() => { const replace = (name: string, attempt: () => void) => { - try { attempt(); } catch { attacks.push(name); } + try { + attempt(); + } catch { + attacks.push(name); + } }; - replace('own_respond', () => Object.defineProperty(generation, 'respond', { value: delegate.provider.respond })); - replace('own_prepare', () => Object.defineProperty(generation, 'prepare', { value: delegate.provider.prepare })); - replace('prototype_swap', () => Object.setPrototypeOf(generation, {})); - replace('prototype_respond', () => Object.defineProperty(BudgetedFixedTraceProvider.prototype, 'respond', { value: delegate.provider.respond })); - replace('prototype_prepare', () => Object.defineProperty(BudgetedFixedTraceProvider.prototype, 'prepare', { value: delegate.provider.prepare })); + replace("own_respond", () => + Object.defineProperty(generation, "respond", { + value: delegate.provider.respond, + }), + ); + replace("own_prepare", () => + Object.defineProperty(generation, "prepare", { + value: delegate.provider.prepare, + }), + ); + replace("prototype_swap", () => Object.setPrototypeOf(generation, {})); + replace("prototype_respond", () => + Object.defineProperty(BudgetedFixedTraceProvider.prototype, "respond", { + value: delegate.provider.respond, + }), + ); + replace("prototype_prepare", () => + Object.defineProperty(BudgetedFixedTraceProvider.prototype, "prepare", { + value: delegate.provider.prepare, + }), + ); }); const budget = new FixedTraceBudget(1); - const policy = fixedTraceResponsePricingPolicy('anthropic', MODEL, PRICING); - const router = new BudgetedFixedTraceProvider(delegate.provider, budget, PRICING, policy); - generation = new BudgetedFixedTraceProvider(delegate.provider, budget, PRICING, policy); + const policy = fixedTraceResponsePricingPolicy("anthropic", MODEL, PRICING); + const router = new BudgetedFixedTraceProvider( + delegate.provider, + budget, + PRICING, + policy, + ); + generation = new BudgetedFixedTraceProvider( + delegate.provider, + budget, + PRICING, + policy, + ); const generationStage = stage(generation, PRICING); generationStage.maxIterations = 2; const artifact = await runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: stage(router, PRICING), generation: generationStage }], + plans: [ + { + name: "anthropic", + router: stage(router, PRICING), + generation: generationStage, + }, + ], baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), - toolDefinitions: canonicalFixedTraceToolDefinitions().filter((tool) => ['search_docs', 'get_doc'].includes(tool.name)), - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', + toolDefinitions: canonicalFixedTraceToolDefinitions().filter((tool) => + ["search_docs", "get_doc"].includes(tool.name), + ), + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", }, budget, outputReservation: reserveFixedTraceDiagnosticOutput(path), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", }); - expect(attacks).toEqual(['own_respond', 'own_prepare', 'prototype_swap', 'prototype_respond', 'prototype_prepare']); + expect(attacks).toEqual([ + "own_respond", + "own_prepare", + "prototype_swap", + "prototype_respond", + "prototype_prepare", + ]); expect(delegate.calls).toHaveLength(3); expect(artifact.runs[0].observations[0].metadata).toMatchObject({ router: { dispatchedCalls: 1, estimatedCostUsd: 0.000035 }, generation: { dispatchedCalls: 2, estimatedCostUsd: 0.00007 }, }); - expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBeCloseTo(0.000105); + expect(artifact.runs[0].summary.totalEstimatedCostUsd).toBeCloseTo( + 0.000105, + ); expect(artifact.budget).toMatchObject({ - dispatchedCalls: 3, completedCalls: 3, budgetRejectedCalls: 0, exposureUnknown: false, + dispatchedCalls: 3, + completedCalls: 3, + budgetRejectedCalls: 0, + exposureUnknown: false, }); expect(artifact.budget.accountedSpendUsd).toBeCloseTo(0.000105); }); - it('rejects a zero-rate ledger with preexisting completed, unknown, or rejected activity', async () => { - const selectedTrace = FIXED_TRACE_SUITE.find((trace) => trace.id === 'surface-channel-chatter'); - if (!selectedTrace) throw new Error('Missing synthetic surface trace'); + it("rejects a zero-rate ledger with preexisting completed, unknown, or rejected activity", async () => { + const selectedTrace = FIXED_TRACE_SUITE.find( + (trace) => trace.id === "surface-channel-chatter", + ); + if (!selectedTrace) throw new Error("Missing synthetic surface trace"); const invoke = async (budget: FixedTraceBudget, suffix: string) => { - const directory = mkdtempSync(join(tmpdir(), 'fixed-trace-output-')); + const directory = mkdtempSync(join(tmpdir(), "fixed-trace-output-")); const provider = scriptedRouter(); - await expect(runFixedTraceDiagnosticArtifact({ - plans: [{ name: 'anthropic', router: budgetedStage(provider.provider, budget), generation: budgetedStage(provider.provider, budget) }], - baseConfig: { - sourceBundleSha256: 'a'.repeat(64), gitCommit: 'abcdef0', gitDirty: false, - promptConfigVersion: 'synthetic-manual-prompt-v1', traceSuite: [selectedTrace], - traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), toolDefinitions: [], - toolDefinitionProvenance: 'fixture_local', architectureArm: 'two_stage_llm_router', - }, - budget, - outputReservation: reserveFixedTraceDiagnosticOutput(join(directory, `${suffix}.json`)), - runRootId: 'root', runStartedAt: '2026-09-05T00:00:00.000Z', - sourceBundleFiles: ['synthetic.ts'], budgetNote: 'Synthetic no-network budget note.', - })).rejects.toThrow('budget must be pristine and exclusively claimed'); + await expect( + runFixedTraceDiagnosticArtifact({ + plans: [ + { + name: "anthropic", + router: budgetedStage(provider.provider, budget), + generation: budgetedStage(provider.provider, budget), + }, + ], + baseConfig: { + sourceBundleSha256: "a".repeat(64), + gitCommit: "abcdef0", + gitDirty: false, + promptConfigVersion: "synthetic-manual-prompt-v1", + traceSuite: [selectedTrace], + traceSuiteSha256: fixedTraceSuiteSha256([selectedTrace]), + toolDefinitions: [], + toolDefinitionProvenance: "fixture_local", + architectureArm: "two_stage_llm_router", + }, + budget, + outputReservation: reserveFixedTraceDiagnosticOutput( + join(directory, `${suffix}.json`), + ), + runRootId: "root", + runStartedAt: "2026-09-05T00:00:00.000Z", + sourceBundleFiles: ["synthetic.ts"], + budgetNote: "Synthetic no-network budget note.", + }), + ).rejects.toThrow("budget must be pristine and exclusively claimed"); expect(provider.calls).toHaveLength(0); }; const prepared = scriptedRouter().provider.prepare(DIAGNOSTIC_TEST_REQUEST); const completed = new FixedTraceBudget(1); - const completedReservation = completed.reserve(prepared, 1, ZERO_RATE_PRICING); + const completedReservation = completed.reserve( + prepared, + 1, + ZERO_RATE_PRICING, + ); completed.markDispatched(completedReservation); - completed.complete(completedReservation, { inputTokens: 1, outputTokens: 1 }, ZERO_RATE_PRICING); - await invoke(completed, 'completed'); + completed.complete( + completedReservation, + { inputTokens: 1, outputTokens: 1 }, + ZERO_RATE_PRICING, + ); + await invoke(completed, "completed"); const unknown = new FixedTraceBudget(1); const unknownReservation = unknown.reserve(prepared, 1, ZERO_RATE_PRICING); unknown.markDispatched(unknownReservation); unknown.markExposureUnknown(unknownReservation); - await invoke(unknown, 'unknown'); + await invoke(unknown, "unknown"); const rejected = new FixedTraceBudget(0.000001); expect(() => rejected.reserve(prepared, 1, PRICING)).toThrow(); - await invoke(rejected, 'rejected'); + await invoke(rejected, "rejected"); }); }); diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts new file mode 100644 index 0000000000..0f42050bef --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -0,0 +1,409 @@ +import { describe, expect, it } from "vitest"; +import { + FIXED_TRACE_ADMITTED_CELLS, + FIXED_TRACE_ARCHITECTURE_CELL_TRUTH, + FIXED_TRACE_COMPONENT_SMOKE_PLAN, + FIXED_TRACE_CONFIRMATORY_POWER_GATE, + FIXED_TRACE_CONFIRMATORY_ADMISSION, + FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT, + FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS, + FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE, + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + FIXED_TRACE_PROTOCOL_PRICING, + FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT, + FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, + assertPromotionGradeDualJudgeFeasibility, + assertFixedTraceEvaluationProtocol, + estimateFixedTraceEvaluationProtocol, + fixedTraceEvaluationProtocolFingerprint, + providerExcludingCalibratedJudges, + selectFixedTraceScreeningSurvivors, + semanticJudgeCandidateProviders, +} from "../../../src/addie/eval/fixed-trace-evaluation-protocol.js"; +import { + FIXED_TRACE_PARTITION_MANIFEST, + assertFixedTracePartitionManifest, +} from "../../../src/addie/eval/fixed-trace-partition.js"; +import { resolveModelCostPricing } from "../../../src/addie/model-cost-pricing.js"; + +const screeningResult = (cell = FIXED_TRACE_ADMITTED_CELLS[0]!, index = 0) => ({ + cellId: cell.id, + role: cell.role, + provider: cell.provider, + model: cell.model, + effort: cell.effort, + configFingerprint: FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT, + safetyFailures: 0, + identityFailures: 0, + malformedFailures: 0, + toolLoopFailures: 0, + reliabilityFailures: index, + latencyMs: 100 + index, + costUsd: index, +}); + +describe("fixed-trace staged protocol", () => { + it("derives the complete 46 development / 36 tuning partitions from corpus authority", () => { + assertFixedTracePartitionManifest(); + expect(FIXED_TRACE_PARTITION_MANIFEST.development).toHaveLength(46); + expect(FIXED_TRACE_PARTITION_MANIFEST.tuning).toHaveLength(36); + expect( + new Set([ + ...FIXED_TRACE_PARTITION_MANIFEST.development, + ...FIXED_TRACE_PARTITION_MANIFEST.tuning, + ]).size, + ).toBe(82); + }); + it("screens every reviewed adapter-supported provider/model/effort cell before adaptive pruning", () => { + expect(FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "router")).toHaveLength(10); + expect(FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "generation")).toHaveLength(11); + for (const role of ["router", "generation"] as const) + for (const provider of ["anthropic", "openai", "google"] as const) + expect( + FIXED_TRACE_ADMITTED_CELLS.some( + (cell) => cell.role === role && cell.provider === provider, + ), + ).toBe(true); + expect( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.provider === "openai", + ).map((cell) => cell.effort), + ).toContain("high"); + expect( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.provider === "google", + ).map((cell) => cell.effort), + ).toContain("medium"); + expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.adaptiveRule).toMatchObject( + { + smokeCases: 8, + developmentCases: 46, + tuningCases: 36, + selection: "predeclared_pareto_successive_halving", + repeats: "stability_only_not_new_cases", + }, + ); + expect(FIXED_TRACE_COMPONENT_SMOKE_PLAN).toMatchObject({ + status: "not_admitted_pending_credential_free_admission", + totalComponentCells: 21, + cases: 8, + repetitions: 1, + maxGenerationInvocationsPerCase: 2, + providerCeilingUsd: 5, + llmJudging: "none", + architectureClaim: "none", + }); + expect(FIXED_TRACE_ARCHITECTURE_CELL_TRUTH).toEqual({ + routerCells: 10, + generationCells: 11, + directCombinations: 11, + twoStageCombinations: 110, + hybridCombinations: 110, + totalArchitectureCombinations: 231, + potentiallyLlmJudgeableProviderMatchedCombinations: 97, + mixedProviderCombinationsRequiringHumanOrFourthProvider: 134, + }); + }); + it("keeps hybrid router fallback and direct admission explicit in worst-case accounting", () => { + const estimate = estimateFixedTraceEvaluationProtocol(); + expect(estimate.hybridWorstCaseRouterCalls).toBe(138); + expect(estimate.hybridWorstCaseRouterCeilingUsd).toBeCloseTo(0.772248, 10); + const architecture = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + expect( + architecture.arms.find((arm) => arm.architecture === "direct_generation") + ?.admission, + ).toBe("not_admitted_architecture"); + expect( + estimate.armCallAccounting.find( + (arm) => arm.armId === "direct-locked-finalist", + ), + ).toMatchObject({ evaluable: false, routerCalls: 0, generationCalls: 0 }); + const hybrid = architecture.arms.find( + (arm) => arm.architecture === "deterministic_policy_llm_fallback_hybrid", + )!; + expect(hybrid.stages.some((stage) => stage.role === "router")).toBe(true); + expect(hybrid.admission).toBe("not_evaluable_no_treatment_contrast"); + expect(FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT).toEqual([ + expect.objectContaining({ + phase: "development", + totalCases: 46, + localTerminalCases: 0, + routedCases: 46, + evaluable: false, + blocker: "no_hybrid_treatment_contrast", + }), + expect.objectContaining({ + phase: "tuning", + totalCases: 36, + localTerminalCases: 0, + routedCases: 36, + evaluable: false, + blocker: "no_hybrid_treatment_contrast", + }), + ]); + expect( + estimate.armCallAccounting.find( + (arm) => arm.armId === "hybrid-locked-finalist", + ), + ).toMatchObject({ + evaluable: false, + localTerminalCases: 0, + routedCases: 138, + routerCalls: 138, + generationCalls: 1_656, + routerCeilingUsd: 0.772248, + }); + }); + it("has no fictional final N and binds named Holm hypotheses", () => { + expect( + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.finalProtocol, + ).toMatchObject({ + status: "unavailable", + externalPackDigest: null, + externalN: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, + fingerprint: null, + powerResult: null, + }); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.requiredIndependentEvaluableCases, + ).toBe(10_562); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.superiorityRequiredIndependentEvaluableCases, + ).toBe(3_803); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.hypotheses.map( + (hypothesis) => hypothesis.id, + ), + ).toEqual([ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]); + expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE).toMatchObject({ + targetPower: 0.8, + conservativeDiscordanceVarianceUpperBound: 1, + worstCaseUpperBoundsNotFinalN: true, + hypotheses: [ + { + id: "H1-superiority", + marginPercentagePoints: 0, + exactTest: "exact_conditional_mcnemar_zero_margin_only", + }, + { + id: "H2-quality-non-inferiority-for-lower-cost-pipeline", + marginPercentagePoints: -3, + exactTest: + "unavailable_pending_independent_Lloyd_Moldovan_score_statistic_E_plus_M_exact_unconditional_noninferiority_implementation_and_type_I_error_validation", + }, + ], + }); + expect(FIXED_TRACE_CONFIRMATORY_ADMISSION).toMatchObject({ + status: "not_admitted_missing_fingerprinted_statistical_protocol", + holm: { K: 2, oneSidedFamilyAlpha: 0.025 }, + unitOfAnalysis: "unique_conversation_user_episode", + repeatedAndTemplateRelatedObservationRule: + "cluster_by_conversation_user_episode; repetitions_never_increase_N", + sizingPilot: { + heldOutFromFinal: true, + reusableInFinal: false, + conservativeDiscordanceUpperBound: null, + }, + judgeCalibration: { + allowedRelationshipToScoredDevelopment: "separate_or_cross_fitted_only", + }, + finalProtocolFingerprint: null, + }); + expect(FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE).toMatchObject({ + status: + "not_admitted_pending_complete_trusted_usage_and_predeclared_economic_margin", + endpoint: "paired_metered_USD_cost_and_latency_reliability", + qualityHypothesisRelationship: "separate_from_H2_quality_noninferiority", + binaryPercentagePointHypothesis: false, + }); + }); + it("fails closed until every provider-excluding judge has a custodied calibration", () => { + for (const provider of ["anthropic", "openai", "google"] as const) { + expect(() => providerExcludingCalibratedJudges([provider])).toThrow( + "no admissible provider-excluding judge pair", + ); + } + expect(FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS).toHaveLength(3); + expect( + FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS.every( + (judge) => + judge.calibrationCorpusSha256 === null && + judge.humanLabelsSha256 === null && + judge.outcomesSha256 === null && + judge.authenticatedAdmission === null && + judge.status === "blocked_pending_authenticated_calibration", + ), + ).toBe(true); + }); + it("fails promotion-grade dual-LLM judging for a mixed router/generator pipeline", () => { + const router = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => + cell.id === "router:anthropic:claude-haiku-4-5:provider_default", + )!; + const generator = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:openai:gpt-5.6-luna:none", + )!; + expect( + semanticJudgeCandidateProviders({ + stages: [ + { role: "router", cellId: router.id }, + { role: "generation", cellId: generator.id }, + ], + } as any), + ).toEqual(["anthropic", "openai"]); + expect(() => + assertPromotionGradeDualJudgeFeasibility({ + stages: [ + { role: "router", cellId: router.id }, + { role: "generation", cellId: generator.id }, + ], + } as any), + ).toThrow("single-provider complete pipeline"); + expect(() => + semanticJudgeCandidateProviders({ + stages: [{ role: "router", cellId: router.id }], + } as any), + ).toThrow("admitted generation provider"); + }); + it("fails closed if a nominally promotion-grade pipeline is made mixed-provider", () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + const architecture = protocol.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + const routed = architecture.arms.find( + (arm) => arm.id === "routed-locked-finalist", + )!; + (routed.stages[0] as { cellId: string }).cellId = + "router:openai:gpt-5.6-luna:none"; + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow( + "pinned declaration", + ); + }); + it("applies hard elimination and successive halving deterministically", () => { + const results = FIXED_TRACE_ADMITTED_CELLS.map(screeningResult); + results[20]!.safetyFailures = 1; + expect(selectFixedTraceScreeningSurvivors(results)).toEqual( + results.slice(0, 10).map((result) => result.cellId), + ); + expect(selectFixedTraceScreeningSurvivors([...results].reverse())).toEqual( + results.slice(0, 10).map((result) => result.cellId), + ); + }); + it("rejects partial and hostile screening result sets", () => { + const result = screeningResult(); + expect(() => selectFixedTraceScreeningSurvivors([result])).toThrow( + "exactly one result for every supported executable cell", + ); + const complete = FIXED_TRACE_ADMITTED_CELLS.map(screeningResult); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, latencyMs: Number.NaN } : entry), + )).toThrow("non-finite number"); + expect(() => selectFixedTraceScreeningSurvivors( + [...complete.slice(0, -1), complete[0]!], + )).toThrow("unknown, duplicate, or mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, provider: "google" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, role: "generation" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, model: "forged-model" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, effort: "high" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, cellId: "alias" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, configFingerprint: "forged" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, forged: true } : entry), + )).toThrow("extra or missing fields"); + }); + it.each([ + (protocol: any) => { protocol.finalProtocol.familywiseAlpha = 0.5; }, + (protocol: any) => { protocol.finalProtocol.hypothesisIds[0] = "rewritten"; }, + (protocol: any) => { protocol.finalProtocol.pairedTest = "rewritten"; }, + (protocol: any) => { protocol.finalProtocol.exclusions = "drop_failures"; }, + (protocol: any) => { protocol.adaptiveRule.repeats = "inflate_N"; }, + (protocol: any) => { protocol.finalProtocol.powerResult = { admitted: true }; }, + (protocol: any) => { protocol.phases[4].arms[0].stages[0].maxOutputTokens = 1; }, + ])("rejects hostile nested protocol rewrites before fingerprinting", (mutate) => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + mutate(protocol); + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow(); + }); + it("rejects getters and proxies before protocol validation, fingerprinting, or budgeting", () => { + const getterProtocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + Object.defineProperty(getterProtocol.adaptiveRule, "repeats", { + enumerable: true, + get: () => "stability_only_not_new_cases", + }); + for (const action of [ + () => assertFixedTraceEvaluationProtocol(getterProtocol), + () => fixedTraceEvaluationProtocolFingerprint(getterProtocol), + () => estimateFixedTraceEvaluationProtocol(getterProtocol), + ]) expect(action).toThrow("own enumerable data property"); + const proxy = new Proxy(structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL), {}); + expect(() => estimateFixedTraceEvaluationProtocol(proxy)).toThrow("must not contain a Proxy"); + const togglingProtocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + let reads = 0; + Object.defineProperty(togglingProtocol.adaptiveRule, "repeats", { + enumerable: true, + get: () => (++reads === 1 ? "stability_only_not_new_cases" : "999"), + }); + expect(() => fixedTraceEvaluationProtocolFingerprint(togglingProtocol)) + .toThrow("own enumerable data property"); + expect(reads).toBe(0); + }); + it("uses a detached protocol snapshot rather than a later nested mutation", () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + const estimate = estimateFixedTraceEvaluationProtocol(protocol); + protocol.phases[5].repetitions = 999; + expect(estimate.stages.find((stage) => stage.phaseId === "stage_4_tuning")?.calls) + .not.toBe(36 * 999); + expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow("pinned declaration"); + }); + it("uses canonical Luna subset-cache pricing and leaves Terra/Sol inert", () => { + const luna = FIXED_TRACE_PROTOCOL_PRICING.find( + (profile) => profile.provider === "openai", + )!; + expect(luna.cacheReadAccounting).toBe("subset"); + expect(luna.cacheReadUsdPerMillionTokens).toBe(0.02); + expect( + resolveModelCostPricing("openai", "gpt-5.6-luna")?.estimateCostMicros({ + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 1_000_000, + }), + ).toBe(20_000); + expect( + FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES.every( + (candidate) => candidate.trustedPrice === null, + ), + ).toBe(true); + }); + it("rejects changing the unadmitted direct boundary or final availability", () => { + const direct = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + direct.phases[4].arms[2].admission = "admitted_diagnostic"; + expect(() => assertFixedTraceEvaluationProtocol(direct)).toThrow( + "not_admitted_architecture", + ); + const final = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + (final.finalProtocol as { externalN: number | null }).externalN = 38; + expect(() => assertFixedTraceEvaluationProtocol(final)).toThrow( + "external final is unavailable", + ); + }); +}); diff --git a/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts new file mode 100644 index 0000000000..c2304dca83 --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-evaluator-coordinator.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from "vitest"; +import { + createFixedTraceEvaluatorCoordinator, + FixedTraceLedgerValidationError, + type FixedTraceActualInvocation, + type FixedTraceExpectedInvocation, +} from "../../../src/addie/eval/fixed-trace-evaluator-coordinator.js"; + +const coordinator = createFixedTraceEvaluatorCoordinator({ + hmacKey: new Uint8Array(32).fill(7), + keyId: "test-evaluator-custody-v1", +}); +const expected = ( + caseId: string, + invocation: number, +): FixedTraceExpectedInvocation => ({ + runId: "run-1", + phaseId: "stage_1_smoke", + caseId, + armId: "arm-1", + stage: "generation", + invocation, + attempt: 1, + requested: { + provider: "anthropic", + model: "claude-sonnet-5", + effort: "provider_default", + identityPolicy: "exact_model_identity_v1", + }, + controls: { + promptSha256: "a", + systemSha256: "b", + messagesSha256: "c", + toolSchemaSha256: "d", + providerRequestSha256: "e", + presentedToolNames: ["search_docs"], + presentedToolOrderSha256: "f", + simulatorReceiptProvenanceSha256: "g", + simulatorControlsSha256: "h", + architectureSha256: "i", + admissionSha256: "j", + configSha256: "k", + pricingSha256: "l", + limitsSha256: "m", + retryCacheSamplingSha256: "n", + failureDenominatorId: "all-planned-invocations-v1", + }, +}); +const actual = ( + entry: FixedTraceExpectedInvocation, +): FixedTraceActualInvocation => ({ + ...entry, + returned: { + provider: "anthropic", + model: "claude-sonnet-5", + identityPolicy: "exact_model_identity_v1", + }, + toolCallsSha256: "o", + toolInputsSha256: "p", + toolResultsSha256: "q", + startedAt: "2026-09-05T00:00:00.000Z", + finishedAt: "2026-09-05T00:00:01.000Z", + latencyMs: 1_000, + usage: { + inputTokens: 1, + outputTokens: 1, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + pricing: { profileId: "p", costUsd: 0.000001 }, + terminalStatus: "complete", + errorCode: null, +}); +const contract = () => + coordinator.issueExpectedSequence({ + runId: "run-1", + protocolFingerprint: "protocol", + manifestFingerprint: "manifest", + entries: [expected("case-a", 1), expected("case-b", 2)], + }); + +describe("fixed-trace evaluator-owned evidence coordinator", () => { + it("authenticates and validates the complete pre-dispatch sequence", () => { + const issued = contract(); + const ledger = coordinator.validate(issued, issued.entries.map(actual)); + expect(ledger).toMatchObject({ + complete: true, + admission: + "not_admitted_diagnostic_hmac_without_privileged_durable_authority", + plannedDenominator: 2, + observedDenominator: 2, + hardFailureDenominator: 0, + }); + expect(issued.keyId).toBe("test-evaluator-custody-v1"); + expect(ledger.signature).toMatch(/^[a-f0-9]{64}$/); + }); + it("snapshots and freezes nested contracts and ledgers", () => { + const entries = [expected("case-a", 1), expected("case-b", 2)]; + const issued = coordinator.issueExpectedSequence({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", entries, + }); + (entries[0]!.controls.presentedToolNames as unknown as string[])[0] = "rewritten"; + expect(issued.entries[0]!.controls.presentedToolNames[0]).toBe("search_docs"); + const supplied = issued.entries.map((entry) => ({ + ...actual(entry), + controls: { + ...entry.controls, + presentedToolNames: [...entry.controls.presentedToolNames], + }, + })); + const ledger = coordinator.validate(issued, supplied); + (supplied[0]!.controls.presentedToolNames as unknown as string[])[0] = "rewritten-again"; + expect(ledger.entries[0]!.controls.presentedToolNames[0]).toBe("search_docs"); + expect(Object.isFrozen(ledger.entries[0]!.controls.presentedToolNames)).toBe(true); + expect(() => { + (ledger.entries[0]!.controls.presentedToolNames as unknown as string[])[0] = "tamper"; + }).toThrow(); + }); + it("never represents caller-keyed HMAC output as privileged custody", () => { + const arbitraryImporter = createFixedTraceEvaluatorCoordinator({ + hmacKey: new Uint8Array(32).fill(9), keyId: "arbitrary-importer-key", + }); + expect(arbitraryImporter.admission).toBe( + "not_admitted_diagnostic_hmac_without_privileged_durable_authority", + ); + const issued = arbitraryImporter.issueExpectedSequence({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", + entries: [expected("case-a", 1)], + }); + expect(arbitraryImporter.validate(issued, [actual(issued.entries[0]!)]).admission) + .toBe("not_admitted_diagnostic_hmac_without_privileged_durable_authority"); + }); + it("rejects getter/proxy inputs and detaches mutable key material", () => { + const key = new Uint8Array(32).fill(3); + const config = { hmacKey: key, keyId: "detached-key" }; + const detached = createFixedTraceEvaluatorCoordinator(config); + key.fill(4); + config.keyId = "rewritten-key"; + const issued = detached.issueExpectedSequence({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", + entries: [expected("case-a", 1)], + }); + expect(issued.keyId).toBe("detached-key"); + expect(detached.validate(issued, [actual(issued.entries[0]!)]).complete).toBe(true); + const getterInput = { + protocolFingerprint: "protocol", manifestFingerprint: "manifest", entries: [expected("case-a", 1)], + } as Record; + let reads = 0; + Object.defineProperty(getterInput, "runId", { + enumerable: true, + get: () => (++reads === 1 ? "run-1" : "run-2"), + }); + expect(() => detached.issueExpectedSequence(getterInput as any)).toThrow("own enumerable data property"); + expect(reads).toBe(0); + expect(() => createFixedTraceEvaluatorCoordinator(new Proxy(config, {}))).toThrow("non-proxy"); + expect(() => detached.issueExpectedSequence(new Proxy({ + runId: "run-1", protocolFingerprint: "protocol", manifestFingerprint: "manifest", entries: [expected("case-a", 1)], + }, {}))).toThrow("must not contain a Proxy"); + const actualEntries = [actual(issued.entries[0]!)]; + expect(() => detached.validate(issued, new Proxy(actualEntries, {}))).toThrow("must not contain a Proxy"); + }); + it.each([ + [ + "omission", + (issued: ReturnType) => [actual(issued.entries[1]!)], + ], + [ + "insertion", + (issued: ReturnType) => [ + { ...actual(issued.entries[0]!), caseId: "unplanned" }, + ], + ], + [ + "duplication", + (issued: ReturnType) => [ + actual(issued.entries[0]!), + actual(issued.entries[0]!), + actual(issued.entries[1]!), + ], + ], + [ + "substitution", + (issued: ReturnType) => [ + { + ...actual(issued.entries[0]!), + requested: { ...issued.entries[0]!.requested, model: "wrong-model" }, + }, + ], + ], + [ + "reordering", + (issued: ReturnType) => [ + actual(issued.entries[1]!), + actual(issued.entries[0]!), + ], + ], + ] as const)( + "rejects %s through its distinct validation branch", + (kind, build) => { + const issued = contract(); + try { + coordinator.validate(issued, build(issued)); + } catch (error) { + expect(error).toBeInstanceOf(FixedTraceLedgerValidationError); + expect((error as FixedTraceLedgerValidationError).tamperClass).toBe( + kind, + ); + return; + } + throw new Error("expected ledger validation failure"); + }, + ); + it("rejects contract restamping and halts unknown exposure", () => { + const issued = contract(); + expect(() => + coordinator.validate( + { ...issued, signature: "00".repeat(32) }, + issued.entries.map(actual), + ), + ).toThrow("authentication"); + expect(() => coordinator.validate( + { ...issued, keyId: "wrong-custody-key" }, + issued.entries.map(actual), + )).toThrow("authentication"); + expect(() => + coordinator.validate(issued, [ + { ...actual(issued.entries[0]!), terminalStatus: "unknown_exposure" }, + ]), + ).toThrow("unknown provider exposure"); + }); +}); diff --git a/server/tests/unit/addie/fixed-trace-judge.test.ts b/server/tests/unit/addie/fixed-trace-judge.test.ts index 8722cca55a..52656b14c6 100644 --- a/server/tests/unit/addie/fixed-trace-judge.test.ts +++ b/server/tests/unit/addie/fixed-trace-judge.test.ts @@ -1,23 +1,23 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it } from "vitest"; import { - FIXED_TRACE_MIN_INDEPENDENT_JUDGES, + FIXED_TRACE_JUDGE_CALIBRATION_ADMISSION, buildFixedTraceJudgeRequest, judgeFixedTraceObservation, runIndependentFixedTraceJudges, summarizeFixedTraceJudges, type FixedTraceJudgeConfig, -} from '../../../src/addie/eval/fixed-trace-judge.js'; +} from "../../../src/addie/eval/fixed-trace-judge.js"; import { BudgetedFixedTraceProvider, FixedTraceBudget, fixedTraceResponsePricingPolicy, -} from '../../../src/addie/eval/fixed-trace-budget.js'; +} from "../../../src/addie/eval/fixed-trace-budget.js"; import { FIXED_TRACE_SUITE, FIXED_TRACE_SUITE_VERSION, type FixedTraceModelStageMetadata, type FixedTraceObservation, -} from '../../../src/addie/eval/fixed-trace-suite.js'; +} from "../../../src/addie/eval/fixed-trace-suite.js"; import type { ModelProvider, ModelProviderCapabilities, @@ -26,13 +26,13 @@ import type { ModelRespondOptions, NormalizedModelEvent, PreparedModelInvocation, -} from '../../../src/addie/model-providers/model-provider.js'; +} from "../../../src/addie/model-providers/model-provider.js"; const CAPABILITIES: ModelProviderCapabilities = { streaming: false, structuredOutput: true, reasoning: true, - reasoningEfforts: ['provider_default', 'none', 'low'], + reasoningEfforts: ["provider_default", "none", "low"], customTools: false, providerWebSearch: false, imageInput: false, @@ -40,14 +40,15 @@ const CAPABILITIES: ModelProviderCapabilities = { }; const PRICING = { - profileId: 'openai-gpt-5.6-luna-standard-2026-08-25', + profileId: "openai-gpt-5.6-luna-2026-08-26", inputUsdPerMillionTokens: 0.2, outputUsdPerMillionTokens: 1.2, cacheReadUsdPerMillionTokens: 0.02, cacheWriteUsdPerMillionTokens: null, - cacheReadAccounting: 'subset' as const, - cacheWriteAccounting: 'unsupported' as const, - source: 'OpenAI gpt-5.6-luna standard, checked 2026-08-25.', + cacheReadAccounting: "subset" as const, + cacheWriteAccounting: "unsupported" as const, + source: + "Repository reviewed OpenAI Luna standard pricing, checked 2026-08-26.", }; class ScriptedJudgeProvider implements ModelProvider { @@ -57,7 +58,7 @@ class ScriptedJudgeProvider implements ModelProvider { constructor( readonly id: ModelProviderId, private readonly output: string | string[], - private readonly finishReason: 'stop' | 'length' = 'stop', + private readonly finishReason: "stop" | "length" = "stop", private readonly includeProviderState = false, ) {} @@ -67,7 +68,11 @@ class ScriptedJudgeProvider implements ModelProvider { model: request.model, capabilities: this.capabilities, requestMetadata: request.requestMetadata, - providerRequest: { model: request.model, messages: request.messages, max: request.maxOutputTokens }, + providerRequest: { + model: request.model, + messages: request.messages, + max: request.maxOutputTokens, + }, }; } @@ -80,9 +85,9 @@ class ScriptedJudgeProvider implements ModelProvider { this.dispatches++; const outputs = Array.isArray(this.output) ? this.output : [this.output]; const providerState = { - type: 'provider_state' as const, + type: "provider_state" as const, provider: this.id, - kind: 'thinking', + kind: "thinking", }; const response = { provider: this.id, @@ -90,284 +95,493 @@ class ScriptedJudgeProvider implements ModelProvider { id: `${this.id}-judge-response`, content: [ ...(this.includeProviderState ? [providerState] : []), - ...outputs.map((text) => ({ type: 'text' as const, text })), + ...outputs.map((text) => ({ type: "text" as const, text })), ], finishReason: this.finishReason, providerFinishReason: this.finishReason, usage: { inputTokens: 100, outputTokens: 20 }, }; - yield { type: 'response_start', provider: this.id, model: request.model, id: response.id }; - if (this.includeProviderState) yield { type: 'provider_state', index: 0, state: providerState }; + yield { + type: "response_start", + provider: this.id, + model: request.model, + id: response.id, + }; + if (this.includeProviderState) + yield { type: "provider_state", index: 0, state: providerState }; for (const [index, text] of outputs.entries()) { - yield { type: 'text_delta', index: index + (this.includeProviderState ? 1 : 0), text }; + yield { + type: "text_delta", + index: index + (this.includeProviderState ? 1 : 0), + text, + }; } - yield { type: 'response_complete', response }; + yield { type: "response_complete", response }; } } function stage(provider: ModelProviderId): FixedTraceModelStageMetadata { return { - source: 'provider', + source: "provider", dispatched: true, + dispatchedCalls: 1, requestedProvider: provider, requestedModel: `${provider}-candidate-secret-model`, returnedProvider: provider, returnedModel: `${provider}-candidate-secret-model`, - modelResolution: 'exact', - promptSha256: 'a'.repeat(64), - providerRequestSha256: 'b'.repeat(64), - reasoningEffort: 'none', + providerExposures: [{ + attempt: 1, + preparedProvider: provider, + preparedModel: `${provider}-candidate-secret-model`, + returnedProvider: provider, + returnedModel: `${provider}-candidate-secret-model`, + }], + modelResolution: "exact", + promptSha256: "a".repeat(64), + providerRequestSha256: "b".repeat(64), + reasoningEffort: "none", maxOutputTokens: 300, timeoutMs: 30_000, maxIterations: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', + samplingMode: "provider_no_sampling_control", temperature: null, usageKnown: true, usage: { inputTokens: 1, outputTokens: 1 }, estimatedCostUsd: 0.001, - pricingSource: 'synthetic', + pricingSource: "synthetic", latencyMs: 10, }; } -function observation(traceId: string, provider: ModelProviderId = 'anthropic'): FixedTraceObservation { +function observation( + traceId: string, + provider: ModelProviderId = "anthropic", +): FixedTraceObservation { return { traceId, metadata: { - runId: 'candidate-secret-run-id', + runId: "candidate-secret-run-id", traceSuiteVersion: FIXED_TRACE_SUITE_VERSION, - traceSuiteSha256: 'c'.repeat(64), - sourceBundleSha256: 'd'.repeat(64), - gitCommit: '0123456789abcdef', + traceSuiteSha256: "c".repeat(64), + sourceBundleSha256: "d".repeat(64), + gitCommit: "0123456789abcdef", gitDirty: false, - addieCodeVersion: 'test', - promptConfigVersion: 'test', - toolSchemaSha256: 'e'.repeat(64), + addieCodeVersion: "test", + promptConfigVersion: "test", + toolSchemaSha256: "e".repeat(64), router: stage(provider), generation: stage(provider), }, - terminalStage: 'generation', - terminalStatus: 'complete', + terminalStage: "generation", + terminalStatus: "complete", boundaryReason: null, localReplacementReason: null, - finishReason: 'stop', - output: 'AdCP uses typed tasks between buyer and seller agents.', + finishReason: "stop", + output: "AdCP uses typed tasks between buyer and seller agents.", flagged: false, - route: { action: 'respond', toolSets: ['knowledge'] }, - tools: [{ - name: 'search_docs', - description: 'Search synthetic official documentation.', - input: { query: 'task model' }, - effect: 'read', - policyDisposition: 'allowed', - resultStatus: 'ok', - simulated: true, - }], + route: { action: "respond", toolSets: ["knowledge"] }, + tools: [ + { + name: "search_docs", + description: "Search synthetic official documentation.", + input: { query: "task model" }, + effect: "read", + policyDisposition: "allowed", + resultStatus: "ok", + simulated: true, + }, + ], }; } function config(provider: ModelProvider): FixedTraceJudgeConfig { return { provider, - model: provider.id === 'openai' ? 'gpt-5.6-luna' : `${provider.id}-judge-model`, - reasoningEffort: provider.id === 'google' ? 'low' : 'none', + model: + provider.id === "openai" ? "gpt-5.6-luna" : `${provider.id}-judge-model`, + reasoningEffort: provider.id === "google" ? "low" : "none", maxOutputTokens: 200, timeoutMs: 30_000, pricing: PRICING, }; } -describe('fixed-trace independent judge', () => { - const trace = FIXED_TRACE_SUITE.find((candidate) => candidate.id === 'knowledge-task-model')!; +describe("fixed-trace independent judge", () => { + const trace = FIXED_TRACE_SUITE.find( + (candidate) => candidate.id === "knowledge-task-model", + )!; - it('builds a blinded request without candidate model, provider, or run identity', () => { + it("builds a blinded request without candidate model, provider, or run identity", () => { const candidate = observation(trace.id); const request = buildFixedTraceJudgeRequest(trace, candidate, { - model: 'judge-model', - reasoningEffort: 'none', + model: "judge-model", + reasoningEffort: "none", maxOutputTokens: 200, }); const serialized = JSON.stringify(request); - expect(serialized).not.toContain('candidate-secret'); - expect(serialized).not.toContain('anthropic'); - expect(serialized).not.toContain('Official task lifecycle: if work is asynchronous'); - expect(serialized).toContain('candidate_answer'); - expect(serialized).toContain('Search synthetic official documentation.'); - expect(serialized).toContain('task model'); - expect(request.requestMetadata).toEqual({ purpose: 'fixed_trace_blinded_judge', trace_id: trace.id }); + expect(serialized).not.toContain("candidate-secret"); + expect(serialized).not.toContain("anthropic"); + expect(serialized).not.toContain( + "Official task lifecycle: if work is asynchronous", + ); + expect(serialized).toContain("candidate_answer"); + expect(serialized).toContain("Search synthetic official documentation."); + expect(serialized).toContain("task model"); + expect(request.requestMetadata).toEqual({ + purpose: "fixed_trace_blinded_judge", + trace_id: trace.id, + }); expect(request.outputSchema).toMatchObject({ - name: 'fixed_trace_judge_verdict', + name: "fixed_trace_judge_verdict", strict: true, schema: { - required: ['pass', 'score', 'reason', 'finding'], + required: ["pass", "score", "reason", "finding"], additionalProperties: false, }, }); }); - it('accepts a strict, internally consistent verdict with complete provenance', async () => { - const provider = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct","finding":"The answer matches the executed tool evidence."}'); - const result = await judgeFixedTraceObservation(trace, observation(trace.id), config(provider)); + it("does not dispatch even a strict verdict without custodied calibration", async () => { + const provider = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer matches the executed tool evidence."}', + ); + const result = await judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ); expect(result).toMatchObject({ - status: 'judged', - failureReason: null, - verdict: { pass: true, score: 4, reason: 'correct', finding: 'The answer matches the executed tool evidence.' }, + status: "skipped", + failureReason: "judge_calibration_not_admitted", + verdict: null, metadata: { candidateIdentityMetadataExposed: false, - requestedProvider: 'openai', - returnedProvider: 'openai', - usageKnown: true, + requestedProvider: "openai", + returnedProvider: null, + usageKnown: false, maxIterations: 1, transportRetries: 0, - samplingMode: 'provider_no_sampling_control', + samplingMode: "provider_no_sampling_control", temperature: null, }, }); - expect(result.metadata.promptSha256).toMatch(/^[a-f0-9]{64}$/); - expect(result.metadata.providerRequestSha256).toMatch(/^[a-f0-9]{64}$/); - expect(result.metadata.responseSha256).toMatch(/^[a-f0-9]{64}$/); - expect(result.metadata.estimatedCostUsd).toBeCloseTo(0.000044); + expect(provider.dispatches).toBe(0); + expect(FIXED_TRACE_JUDGE_CALIBRATION_ADMISSION).toBe( + "not_admitted_missing_privileged_custodied_calibration", + ); }); - it('joins a valid verdict split across provider text blocks', async () => { - const provider = new ScriptedJudgeProvider('openai', [ + it("does not process provider text before calibration admission", async () => { + const provider = new ScriptedJudgeProvider("openai", [ '{"pass":true,', '"score":3,"reason":"correct","finding":"The answer is supported."}', ]); - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(provider))) - .resolves.toMatchObject({ - status: 'judged', - verdict: { pass: true, score: 3, reason: 'correct' }, - }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ), + ).resolves.toMatchObject({ + status: "skipped", + failureReason: "judge_calibration_not_admitted", + }); }); - it('accepts a verdict accompanied only by authenticated provider thinking state', async () => { + it("does not process provider state before calibration admission", async () => { const provider = new ScriptedJudgeProvider( - 'anthropic', + "anthropic", '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', - 'stop', + "stop", true, ); - await expect(judgeFixedTraceObservation(trace, observation(trace.id, 'openai'), config(provider))) - .resolves.toMatchObject({ - status: 'judged', - verdict: { pass: true, score: 4, reason: 'correct' }, - }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id, "openai"), + config(provider), + ), + ).resolves.toMatchObject({ + status: "skipped", + failureReason: "judge_calibration_not_admitted", + }); }); - it('rejects inconsistent or truncated judge output', async () => { - const inconsistent = new ScriptedJudgeProvider('openai', '{"pass":true,"score":2,"reason":"correct"}'); - const truncated = new ScriptedJudgeProvider('google', '{"pass":true', 'length'); - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(inconsistent))) - .resolves.toMatchObject({ status: 'invalid', failureReason: 'judge_output_invalid' }); - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(truncated))) - .resolves.toMatchObject({ status: 'invalid', failureReason: 'judge_output_truncated' }); + it("does not dispatch malformed candidate verdicts before calibration admission", async () => { + const inconsistent = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":2,"reason":"correct"}', + ); + const truncated = new ScriptedJudgeProvider( + "google", + '{"pass":true', + "length", + ); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(inconsistent), + ), + ).resolves.toMatchObject({ + status: "skipped", + failureReason: "judge_calibration_not_admitted", + }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(truncated), + ), + ).resolves.toMatchObject({ + status: "skipped", + failureReason: "judge_calibration_not_admitted", + }); }); - it('requires a bounded audit finding in every verdict', async () => { + it("does not dispatch malformed audit findings before calibration admission", async () => { const missing = new ScriptedJudgeProvider( - 'openai', + "openai", '{"pass":true,"score":4,"reason":"correct"}', ); const blank = new ScriptedJudgeProvider( - 'openai', + "openai", '{"pass":true,"score":4,"reason":"correct","finding":""}', ); const oversized = new ScriptedJudgeProvider( - 'openai', - JSON.stringify({ pass: true, score: 4, reason: 'correct', finding: 'x'.repeat(241) }), + "openai", + JSON.stringify({ + pass: true, + score: 4, + reason: "correct", + finding: "x".repeat(241), + }), ); for (const provider of [missing, blank, oversized]) { - await expect(judgeFixedTraceObservation(trace, observation(trace.id), config(provider))) - .resolves.toMatchObject({ status: 'invalid', failureReason: 'judge_output_invalid' }); + await expect( + judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ), + ).resolves.toMatchObject({ + status: "skipped", + failureReason: "judge_calibration_not_admitted", + }); } }); - it('refuses a same-provider judge before dispatch', async () => { - const provider = new ScriptedJudgeProvider('anthropic', '{"pass":true,"score":4,"reason":"correct"}'); - const result = await judgeFixedTraceObservation(trace, observation(trace.id), config(provider)); - expect(result).toMatchObject({ status: 'skipped', failureReason: 'judge_not_independent' }); + it("refuses a same-provider judge before dispatch", async () => { + const provider = new ScriptedJudgeProvider( + "anthropic", + '{"pass":true,"score":4,"reason":"correct"}', + ); + const result = await judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), + ); + expect(result).toMatchObject({ + status: "skipped", + failureReason: "judge_not_independent", + }); expect(provider.dispatches).toBe(0); }); - it('also excludes a returned fallback provider from the judge panel', async () => { + it("also excludes a returned fallback provider from the judge panel", async () => { const candidate = observation(trace.id); - candidate.metadata.generation.returnedProvider = 'google'; - candidate.metadata.generation.returnedModel = 'google-fallback-secret-model'; - candidate.metadata.generation.modelResolution = 'provider_canonicalized'; - const provider = new ScriptedJudgeProvider('google', '{"pass":true,"score":4,"reason":"correct"}'); - const result = await judgeFixedTraceObservation(trace, candidate, config(provider)); - expect(result).toMatchObject({ status: 'skipped', failureReason: 'judge_not_independent' }); + candidate.metadata.generation.returnedProvider = "google"; + candidate.metadata.generation.returnedModel = + "google-fallback-secret-model"; + candidate.metadata.generation.modelResolution = "provider_canonicalized"; + candidate.metadata.generation.providerExposures = [{ + attempt: 1, + preparedProvider: "anthropic", + preparedModel: "anthropic-candidate-secret-model", + returnedProvider: "google", + returnedModel: "google-fallback-secret-model", + }]; + const provider = new ScriptedJudgeProvider( + "google", + '{"pass":true,"score":4,"reason":"correct"}', + ); + const result = await judgeFixedTraceObservation( + trace, + candidate, + config(provider), + ); + expect(result).toMatchObject({ + status: "skipped", + failureReason: "judge_not_independent", + }); + expect(provider.dispatches).toBe(0); + }); + + it("unions requested and returned router and generator providers for pipeline exclusion", async () => { + const candidate = observation(trace.id, "anthropic"); + candidate.metadata.router.returnedProvider = "openai"; + candidate.metadata.router.requestedModel = "anthropic-router"; + candidate.metadata.router.returnedModel = "openai-router-fallback"; + candidate.metadata.generation.requestedProvider = "google"; + candidate.metadata.generation.requestedModel = "google-generator"; + candidate.metadata.generation.returnedProvider = "google"; + candidate.metadata.generation.returnedModel = "google-generator-fallback"; + candidate.metadata.router.providerExposures = [{ + attempt: 1, + preparedProvider: "anthropic", + preparedModel: "anthropic-router", + returnedProvider: "openai", + returnedModel: "openai-router-fallback", + }]; + candidate.metadata.generation.providerExposures = [{ + attempt: 1, + preparedProvider: "google", + preparedModel: "google-generator", + returnedProvider: "google", + returnedModel: "google-generator-fallback", + }]; + const onlyRemainingProvider = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', + ); + const sameRouterProvider = new ScriptedJudgeProvider( + "anthropic", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', + ); + await expect(judgeFixedTraceObservation(trace, candidate, config(sameRouterProvider))) + .resolves.toMatchObject({ status: "skipped", failureReason: "judge_not_independent" }); + await expect(runIndependentFixedTraceJudges( + [trace], [candidate], [config(onlyRemainingProvider)], + )).rejects.toThrow("privileged custodied calibration"); + expect(sameRouterProvider.dispatches).toBe(0); + expect(onlyRemainingProvider.dispatches).toBe(0); + }); + + it("fails closed when an LLM-contributing stage has no exposure ledger", async () => { + const candidate = observation(trace.id); + delete candidate.metadata.router.providerExposures; + const provider = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"must not dispatch"}', + ); + await expect(judgeFixedTraceObservation(trace, candidate, config(provider))) + .resolves.toMatchObject({ + status: "skipped", + failureReason: "candidate_not_judgeable", + }); expect(provider.dispatches).toBe(0); }); - it('attributes a budget rejection without dispatching the judge', async () => { - const delegate = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct"}'); + it("fails closed when a terminal or exposure provider identity is unknown or unledgered", async () => { + const terminalMismatch = observation(trace.id); + terminalMismatch.metadata.router.returnedProvider = "openai"; + terminalMismatch.metadata.router.returnedModel = "openai-hidden-fallback"; + const unknownExposure = observation(trace.id) as any; + unknownExposure.metadata.generation.providerExposures[0].returnedProvider = "unknown"; + const provider = new ScriptedJudgeProvider( + "google", + '{"pass":true,"score":4,"reason":"correct","finding":"must not dispatch"}', + ); + for (const candidate of [terminalMismatch, unknownExposure]) { + await expect(judgeFixedTraceObservation(trace, candidate, config(provider))) + .resolves.toMatchObject({ + status: "skipped", + failureReason: "candidate_not_judgeable", + }); + } + expect(provider.dispatches).toBe(0); + }); + + it("blocks a budgeted judge before any provider exposure without calibration", async () => { + const delegate = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct"}', + ); const budget = new FixedTraceBudget(0.000001); const provider = new BudgetedFixedTraceProvider( delegate, budget, PRICING, - fixedTraceResponsePricingPolicy('openai', 'gpt-5.6-luna', PRICING), + fixedTraceResponsePricingPolicy("openai", "gpt-5.6-luna", PRICING), + ); + const result = await judgeFixedTraceObservation( + trace, + observation(trace.id), + config(provider), ); - const result = await judgeFixedTraceObservation(trace, observation(trace.id), config(provider)); expect(result).toMatchObject({ - status: 'not_dispatched_budget', - failureReason: 'judge_budget_rejected', + status: "skipped", + failureReason: "judge_calibration_not_admitted", metadata: { usageKnown: false, estimatedCostUsd: 0 }, }); - expect(result.metadata.providerRequestSha256).toMatch(/^[a-f0-9]{64}$/); expect(delegate.dispatches).toBe(0); }); - it('requires and summarizes two distinct non-candidate judge providers', async () => { + it("blocks an otherwise independent panel without custodied calibration", async () => { const candidate = observation(trace.id); - const openai = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}'); - const google = new ScriptedJudgeProvider('google', '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}'); - const judgments = await runIndependentFixedTraceJudges( - [trace], - [candidate], - [config(openai), config(google)], + const openai = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct","finding":"The answer is supported."}', + ); + const google = new ScriptedJudgeProvider( + "google", + '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}', ); - expect(judgments).toHaveLength(FIXED_TRACE_MIN_INDEPENDENT_JUDGES); - expect(summarizeFixedTraceJudges([trace], [candidate], judgments)).toMatchObject({ + await expect(runIndependentFixedTraceJudges( + [trace], [candidate], [config(openai), config(google)], + )).rejects.toThrow("privileged custodied calibration"); + expect( + summarizeFixedTraceJudges([trace], [candidate], []), + ).toMatchObject({ expectedCases: 1, expectedJudgments: 2, - observedJudgments: 2, - judgedJudgments: 2, - complete: true, - judgmentCoverageRate: 1, - consensusPassRate: 1, - disagreementRate: 0, - comparisonEligible: true, + observedJudgments: 0, + judgedJudgments: 0, + complete: false, + judgmentCoverageRate: 0, + consensusPassRate: null, + disagreementRate: null, + comparisonEligible: false, }); }); - it('rejects an incomplete independent judge panel before any judge dispatch', async () => { - const openai = new ScriptedJudgeProvider('openai', '{"pass":true,"score":4,"reason":"correct"}'); - await expect(runIndependentFixedTraceJudges( - [trace], - [observation(trace.id)], - [config(openai)], - )).rejects.toThrow('requires at least two independent judge providers'); + it("rejects an incomplete independent judge panel before any judge dispatch", async () => { + const openai = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":4,"reason":"correct"}', + ); + await expect( + runIndependentFixedTraceJudges( + [trace], + [observation(trace.id)], + [config(openai)], + ), + ).rejects.toThrow("privileged custodied calibration"); expect(openai.dispatches).toBe(0); }); - it('records disagreement as a failed consensus without hiding completed coverage', async () => { + it("does not score disagreement without custodied calibration", async () => { const candidate = observation(trace.id); - const openai = new ScriptedJudgeProvider('openai', '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}'); - const google = new ScriptedJudgeProvider('google', '{"pass":false,"score":2,"reason":"incomplete","finding":"The answer omits a required criterion."}'); - const judgments = await runIndependentFixedTraceJudges( - [trace], - [candidate], - [config(openai), config(google)], + const openai = new ScriptedJudgeProvider( + "openai", + '{"pass":true,"score":3,"reason":"correct","finding":"The answer is supported."}', ); - expect(summarizeFixedTraceJudges([trace], [candidate], judgments)).toMatchObject({ - judgmentCoverageRate: 1, - consensusPassRate: 0, - disagreementRate: 1, - comparisonEligible: true, + const google = new ScriptedJudgeProvider( + "google", + '{"pass":false,"score":2,"reason":"incomplete","finding":"The answer omits a required criterion."}', + ); + await expect(runIndependentFixedTraceJudges( + [trace], [candidate], [config(openai), config(google)], + )).rejects.toThrow("privileged custodied calibration"); + expect( + summarizeFixedTraceJudges([trace], [candidate], []), + ).toMatchObject({ + judgmentCoverageRate: 0, + consensusPassRate: null, + disagreementRate: null, + comparisonEligible: false, }); }); }); diff --git a/server/tests/unit/addie/fixed-trace-runner.test.ts b/server/tests/unit/addie/fixed-trace-runner.test.ts index e56dfb629d..c11d750f12 100644 --- a/server/tests/unit/addie/fixed-trace-runner.test.ts +++ b/server/tests/unit/addie/fixed-trace-runner.test.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import Ajv from 'ajv'; import { describe, expect, it, vi } from 'vitest'; import { + assertFixedTraceArchitectureComparisonPrerequisite, buildFixedTraceGenerationRequest, fixedTraceArchitectureConfigSha256, fixedTraceToolSchemaSha256, @@ -400,6 +401,15 @@ function expandedFixtureTrace(id = 'expanded-fixture-tool'): FixedTraceCase { } describe('fixed trace artifact runner', () => { + it('fails closed architecture comparison while candidate-visible tools are fixture-local', () => { + const router = new ScriptedProvider([]); + const generation = new ScriptedProvider([]); + expect(() => assertFixedTraceArchitectureComparisonPrerequisite(config(router, generation))).toThrow( + 'common authenticated base registry/schema/receipt tool universe is unavailable', + ); + expect(router.respondCalls).toHaveLength(0); + expect(generation.respondCalls).toHaveLength(0); + }); it('uses production quick-match terminal behavior only from allowed request facts', () => { const selectedTrace = trace('knowledge-task-model'); const policy = fixedTraceHybridPolicy(); @@ -471,6 +481,12 @@ describe('fixed trace artifact runner', () => { expect(ambiguous.terminalStage).toBe('generation'); expect(ambiguousRouter.respondCalls).toHaveLength(1); expect(ambiguousGeneration.respondCalls).toHaveLength(1); + expect(ambiguous.metadata.router.providerExposures).toEqual([ + expect.objectContaining({ attempt: 1, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + ]); + expect(ambiguous.metadata.generation.providerExposures).toEqual([ + expect.objectContaining({ attempt: 1, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + ]); }); it('fails hybrid admission safe for tool-bearing, admin, thread, and unknown-privacy cases', () => { @@ -745,6 +761,10 @@ describe('fixed trace artifact runner', () => { estimatedCostUsd: 0.00007, }); expect(observation.metadata.generation.providerRequestSha256).toMatch(/^[a-f0-9]{64}$/); + expect(observation.metadata.generation.providerExposures).toEqual([ + expect.objectContaining({ attempt: 1, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + expect.objectContaining({ attempt: 2, preparedProvider: 'anthropic', returnedProvider: 'anthropic' }), + ]); expect(generation.respondCalls).toHaveLength(2); expect(generation.respondCalls[0].toolChoice).toEqual({ type: 'tool', name: 'search_docs' }); expect(generation.respondCalls[1].toolChoice).toBeUndefined(); diff --git a/server/tests/unit/addie/model-provider-openai-google.test.ts b/server/tests/unit/addie/model-provider-openai-google.test.ts index 2b58ce9df8..d5519d4ed5 100644 --- a/server/tests/unit/addie/model-provider-openai-google.test.ts +++ b/server/tests/unit/addie/model-provider-openai-google.test.ts @@ -89,6 +89,14 @@ function googleResponse(overrides: Record = {}): GenerateConten } describe('OpenAIResponsesProvider', () => { + it('keeps Terra and Sol outside the production OpenAI dispatch boundary', () => { + const provider = new OpenAIResponsesProvider('unused', {} as OpenAIResponsesTransport); + expect(provider.prepare(request(OPENAI_ROUTER_MODEL)).providerRequest).toMatchObject({ model: OPENAI_ROUTER_MODEL }); + expect(() => provider.prepare(request('gpt-5.6-terra'))).toThrow('Unsupported OpenAI router model'); + expect(() => provider.prepare(request('gpt-5.6-sol'))).toThrow('Unsupported OpenAI router model'); + expect(() => provider.prepare(request('gpt-5.6-terra-20260905'))).toThrow('Unsupported OpenAI router model'); + }); + it.each([ [{ type: 'auto' as const }, 'auto'], [{ type: 'required' as const }, 'required'],