feat(addie): add offline fixed-trace evaluation planning

server/src/addie/direct-tool-universe.ts

@@ -166,6 +166,7 @@ function captureFixedTraceEvaluatorToolUniverse(): CapturedDirectToolUniverse {
  */
 export const FIXED_TRACE_DIRECT_TOOL_UNIVERSE = captureFixedTraceEvaluatorToolUniverse();
 
-export const FIXED_TRACE_DIRECT_TOOL_HANDLERS = createSyntheticDirectToolReceiptHandlers(
-  FIXED_TRACE_DIRECT_TOOL_UNIVERSE,
-);
+/** Construct inert handlers only when a direct replay explicitly asks for them. */
+export function fixedTraceDirectToolHandlers(): Map<string, ToolHandler> {
+  return createSyntheticDirectToolReceiptHandlers(FIXED_TRACE_DIRECT_TOOL_UNIVERSE);
+}

server/src/addie/eval/fixed-trace-diagnostic-cli.ts

@@ -4,10 +4,12 @@ export interface FixedTraceDiagnosticCliArguments {
   suite?: string;
   softMaxUsd?: string;
   output?: string;
+  experimentPlan?: string;
+  trustedManifest?: string;
   validateOnly: boolean;
 }
 
-const NAMES = new Set(['providers', 'architecture-arm', 'suite', 'soft-max-usd', 'output', 'validate-only']);
+const NAMES = new Set(['providers', 'architecture-arm', 'suite', 'soft-max-usd', 'output', 'experiment-plan', 'trusted-manifest', 'validate-only']);
 
 /** Strict, side-effect-free parser for the diagnostic-only manual evaluator. */
 export function parseFixedTraceDiagnosticCliArguments(values: readonly string[]): FixedTraceDiagnosticCliArguments {
@@ -33,6 +35,8 @@ export function parseFixedTraceDiagnosticCliArguments(values: readonly string[])
     suite: typeof seen.get('suite') === 'string' ? seen.get('suite') as string : undefined,
     softMaxUsd: typeof seen.get('soft-max-usd') === 'string' ? seen.get('soft-max-usd') as string : undefined,
     output: typeof seen.get('output') === 'string' ? seen.get('output') as string : undefined,
+    experimentPlan: typeof seen.get('experiment-plan') === 'string' ? seen.get('experiment-plan') as string : undefined,
+    trustedManifest: typeof seen.get('trusted-manifest') === 'string' ? seen.get('trusted-manifest') as string : undefined,
     validateOnly: seen.get('validate-only') === true || seen.get('validate-only') === 'true',
   };
 }

server/src/addie/eval/fixed-trace-diagnostic-run.ts

@@ -23,6 +23,8 @@ import {
   fixedTraceResponsePricingPolicy,
   isTrustedBudgetedFixedTraceProvider,
 } from './fixed-trace-budget.js';
+import { types } from 'node:util';
+import { snapshotFixedTraceJson } from './fixed-trace-safe-snapshot.js';
 
 export interface FixedTraceDiagnosticProviderPlan {
   readonly name: string;
@@ -64,6 +66,17 @@ function ownDataProperty(source: unknown, name: string, owner: string): unknown
   return descriptor.value;
 }
 
+function assertClosedOwnDataRecord(source: unknown, fields: readonly string[], owner: string): void {
+  if (typeof source !== 'object' || source === null || types.isProxy(source) || Object.getPrototypeOf(source) !== Object.prototype) {
+    throw new Error(`Fixed trace diagnostic ${owner} must be a plain non-Proxy object`);
+  }
+  const keys = Reflect.ownKeys(source);
+  if (keys.length !== fields.length || keys.some((key) => typeof key !== 'string' || !fields.includes(key))) {
+    throw new Error(`Fixed trace diagnostic ${owner} must contain exactly its approved fields`);
+  }
+  for (const field of fields) ownDataProperty(source, field, owner);
+}
+
 const DIAGNOSTIC_PRICING_FIELDS = [
   'profileId',
   'inputUsdPerMillionTokens',
@@ -76,19 +89,7 @@ const DIAGNOSTIC_PRICING_FIELDS = [
 ] as const;
 
 function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing {
-  const prototype = typeof pricing === 'object' && pricing !== null
-    ? Object.getPrototypeOf(pricing)
-    : null;
-  if (
-    typeof pricing !== 'object'
-    || pricing === null
-    || (prototype !== Object.prototype && prototype !== null)
-  ) throw new Error(`Fixed trace diagnostic ${owner} must be a plain pricing object`);
-  const keys = Reflect.ownKeys(pricing);
-  if (
-    keys.length !== DIAGNOSTIC_PRICING_FIELDS.length
-    || keys.some((key) => typeof key !== 'string' || !DIAGNOSTIC_PRICING_FIELDS.includes(key as typeof DIAGNOSTIC_PRICING_FIELDS[number]))
-  ) throw new Error(`Fixed trace diagnostic ${owner} must contain only approved pricing fields`);
+  assertClosedOwnDataRecord(pricing, DIAGNOSTIC_PRICING_FIELDS, owner);
   // Structured cloning calls nested getters. Copy each approved data
   // descriptor instead, so a price cannot change between validation and use.
   return Object.freeze({
@@ -106,6 +107,10 @@ function snapshotPricing(pricing: unknown, owner: string): FixedTracePricing {
 function snapshotStageConfig(config: unknown, owner: string): FixedTraceProviderStageConfig {
   // Read each untrusted stage property exactly once. Later checks use only
   // this detached plain object, never a caller-controlled getter or proxy.
+  assertClosedOwnDataRecord(config, [
+    'provider', 'model', 'reasoningEffort', 'maxOutputTokens', 'timeoutMs',
+    'maxIterations', 'transportRetries', 'samplingMode', 'temperature', 'pricing',
+  ], owner);
   const provider = ownDataProperty(config, 'provider', owner);
   const model = ownDataProperty(config, 'model', owner);
   const reasoningEffort = ownDataProperty(config, 'reasoningEffort', owner);
@@ -133,12 +138,7 @@ function snapshotStageConfig(config: unknown, owner: string): FixedTraceProvider
 function snapshotBaseConfig(
   config: FixedTraceDiagnosticArtifactOptions['baseConfig'],
 ): FixedTraceDiagnosticArtifactOptions['baseConfig'] {
-  const { traceSuite, toolDefinitions, ...serializable } = config;
-  return Object.freeze({
-    ...structuredClone(serializable),
-    traceSuite: deepFreeze(structuredClone(traceSuite)),
-    toolDefinitions: deepFreeze(structuredClone(toolDefinitions)),
-  });
+  return snapshotFixedTraceJson(config, 'fixed trace diagnostic base config') as FixedTraceDiagnosticArtifactOptions['baseConfig'];
 }
 
 function snapshotPlans(
@@ -148,6 +148,19 @@ function snapshotPlans(
   if (!Array.isArray(suppliedPlans) || suppliedPlans.length === 0) {
     throw new Error('Fixed trace diagnostic run requires one or more provider plans');
   }
+  if (types.isProxy(suppliedPlans) || Object.getPrototypeOf(suppliedPlans) !== Array.prototype || Object.getOwnPropertySymbols(suppliedPlans).length !== 0) {
+    throw new Error('Fixed trace diagnostic provider plans must be a plain non-Proxy array');
+  }
+  const planDescriptors = Object.getOwnPropertyDescriptors(suppliedPlans);
+  for (const key of Object.keys(planDescriptors)) {
+    if (key === 'length') continue;
+    if (!/^(0|[1-9][0-9]*)$/.test(key) || !('value' in planDescriptors[key]!) || !planDescriptors[key]!.enumerable) {
+      throw new Error('Fixed trace diagnostic provider plans contain an accessor or extra property');
+    }
+  }
+  for (const [index, suppliedPlan] of suppliedPlans.entries()) {
+    assertClosedOwnDataRecord(suppliedPlan, ['name', 'router', 'generation'], `provider plan ${index}`);
+  }
   const plans = Object.freeze(suppliedPlans.map((suppliedPlan, index) => Object.freeze({
     // Do not validate while reading: a plan accessor must not be able to
     // return one identity for validation and another for execution.