diff --git a/server/src/addie/eval/fixed-trace-evaluation-protocol.ts b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts new file mode 100644 index 0000000000..338f2b2170 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-evaluation-protocol.ts @@ -0,0 +1,1402 @@ +import { createHash } from "node:crypto"; +import { + resolveModelCostPricing, +} from "../model-cost-pricing.js"; +import { ANTHROPIC_PROVIDER_CAPABILITIES } from "../model-providers/anthropic-provider.js"; +import { + GOOGLE_GENERATE_CONTENT_CAPABILITIES, + GOOGLE_ROUTER_MODEL, +} from "../model-providers/google-generate-content-provider.js"; +import type { + ModelProviderId, + ModelReasoningEffort, +} from "../model-providers/model-provider.js"; +import { + OPENAI_RESPONSES_CAPABILITIES, + OPENAI_ROUTER_MODEL, +} from "../model-providers/openai-responses-provider.js"; +import { ANTHROPIC_ROUTER_CAPABILITIES } from "../model-providers/anthropic-router-provider.js"; +import { + decideFixedTraceHybridRoute, + fixedTraceHybridPolicy, + type FixedTraceArchitectureArmId, +} from "./fixed-trace-architecture.js"; +import { + FIXED_TRACE_PARTITION_MANIFEST, + assertFixedTracePartitionManifest, +} from "./fixed-trace-partition.js"; +import { assertFixedTraceExperimentalDesign } from "./fixed-trace-experimental-design.js"; +import { snapshotFixedTraceJson } from "./fixed-trace-safe-snapshot.js"; +import { FIXED_TRACE_CORPUS } from "./fixed-trace-suite.js"; + +export const FIXED_TRACE_EVALUATION_PROTOCOL_VERSION = + "addie-fixed-trace-evaluation-protocol-v3" as const; + +export const FIXED_TRACE_CONFIRMATORY_POWER_GATE = Object.freeze({ + version: "addie-fixed-trace-confirmatory-power-v2", + familywiseAlpha: 0.025, + hypotheses: Object.freeze([ + Object.freeze({ + id: "H1-superiority", + comparison: "locked-pipeline-candidate vs locked-pipeline-comparator", + endpoint: "two-judge blinded quality success rate", + direction: "greater", + marginPercentagePoints: 0, + alternativeDifferencePercentagePoints: 5, + oneSidedAlpha: "assigned_by_Holm_to_ordered_p_values_after_locked_gatekeeping_graph", + exactTest: "exact_conditional_mcnemar_zero_margin_only", + }), + Object.freeze({ + id: "H2-quality-non-inferiority-for-lower-cost-pipeline", + comparison: + "lower-metered-cost locked pipeline quality vs locked-pipeline-comparator quality", + endpoint: "two-judge blinded quality success rate", + direction: "not_less_than", + marginPercentagePoints: -3, + alternativeDifferencePercentagePoints: 0, + oneSidedAlpha: "assigned_by_Holm_to_ordered_p_values_after_locked_gatekeeping_graph", + exactTest: + "unavailable_pending_independent_Lloyd_Moldovan_score_statistic_E_plus_M_exact_unconditional_noninferiority_implementation_and_type_I_error_validation", + sensitivityOnly: + "Sidik_exact_CI_or_p_value_after_primary_Lloyd_Moldovan_verification", + }), + ]), + test: "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method", + bootstrap: "grouped_stratified_case_level_bootstrap", + exclusionRule: "hard_failures_and_missing_evidence_remain_in_denominator", + repetitionsCountAsIndependentCases: false, + conservativeNormalApproximationBounds: Object.freeze({ + alpha: 0.0125, + H1Superiority: 3_803, + H2QualityNonInferiority: 10_562, + H2AtDisplayedAlpha025: 8_721, + notExactEMPower: true, + }), + targetPower: 0.8, + planningAlternative: + "H1: +5pp over zero; H2: 0pp, three points above the -3pp NI margin", + conservativeDiscordanceVarianceUpperBound: 1, + worstCaseUpperBoundsNotFinalN: true, + finalNReductionRule: + "only_sealed_never_reused_sizing_pilot_or_predeclared_blinded_arm_invariant_discordance_only_upward_internal_pilot_with_enumerated_adaptive_exact_type_I_error", + externalFinalN: null, + externalFinalStatus: + "unavailable_pending_fingerprinted_exact_paired_discordance_power_result", +} as const); + +export type FixedTraceProtocolPhaseId = + | "stage_0_preflight_calibration" + | "stage_1_smoke" + | "stage_2_router_screen" + | "stage_2_oracle_generator_screen" + | "stage_3_architecture" + | "stage_4_tuning" + | "stage_5_external_final" + | "stage_6_canary"; +export type FixedTraceProtocolStageRole = + "router" | "generation" | "judge" | "simulator"; +export type FixedTraceProtocolAdmission = + | "admitted_diagnostic" + | "not_admitted_dispatch_authority" + | "not_admitted_architecture" + | "not_evaluable_no_treatment_contrast" + | "not_admitted_external_final" + | "not_admitted_canary"; + +/** + * A planning descriptor is deliberately not a copied rate card. The live + * resolver is the sole source of a prospective rate; an effective interval is + * required before a plan can become costed or dispatchable. + */ +export interface FixedTraceProtocolPricingProfile { + readonly provider: ModelProviderId; + readonly model: string; + readonly profileId: string | null; + readonly version: string | null; + readonly effectiveFrom: string | null; + readonly effectiveBefore: string | null; + readonly status: "available" | "unavailable_missing_canonical_price" | "unavailable_missing_effective_interval"; +} + +function canonicalPricingDescriptor( + provider: ModelProviderId, + model: string, +): FixedTraceProtocolPricingProfile { + const pricing = resolveModelCostPricing(provider, model); + if (!pricing) return Object.freeze({ + provider, model, profileId: null, version: null, effectiveFrom: null, + effectiveBefore: null, status: "unavailable_missing_canonical_price", + }); + // The current live registry records only an end date (or no date), not the + // complete dated cohort interval required for a prospective evaluation. + // Therefore no numeric cost or reservation can be inferred from it here. + return Object.freeze({ + provider, + model, + profileId: pricing.version, + version: pricing.version, + effectiveFrom: null, + effectiveBefore: pricing.validBefore?.toISOString() ?? null, + status: "unavailable_missing_effective_interval", + }); +} + +export const FIXED_TRACE_PROTOCOL_PRICING = Object.freeze([ + canonicalPricingDescriptor("anthropic", "claude-haiku-4-5"), + canonicalPricingDescriptor("anthropic", "claude-sonnet-5"), + canonicalPricingDescriptor("openai", OPENAI_ROUTER_MODEL), + canonicalPricingDescriptor("google", GOOGLE_ROUTER_MODEL), +] satisfies readonly FixedTraceProtocolPricingProfile[]); + +export interface FixedTraceAdmittedCell { + readonly id: string; + readonly role: "router" | "generation"; + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly pricingProfileId: string | null; + readonly adapterCapabilitySource: string; +} +const efforts = (values: readonly ModelReasoningEffort[]) => + values.length ? values : ["provider_default" as const]; +const priceId = (provider: ModelProviderId, model: string) => { + const profile = FIXED_TRACE_PROTOCOL_PRICING.find( + (entry) => entry.provider === provider && entry.model === model, + ); + if (!profile) + throw new Error(`No immutable price for admitted ${provider}/${model}`); + return profile.profileId; +}; +const cells = ( + role: "router" | "generation", + provider: ModelProviderId, + model: string, + values: readonly ModelReasoningEffort[], + source: string, +) => + efforts(values).map((effort) => + Object.freeze({ + id: `${role}:${provider}:${model}:${effort}`, + role, + provider, + model, + effort, + pricingProfileId: priceId(provider, model), + adapterCapabilitySource: source, + }), + ); + +/** Derived only from reviewed exported adapter capabilities and immutable prices. */ +export const FIXED_TRACE_ADMITTED_CELLS: readonly FixedTraceAdmittedCell[] = + Object.freeze([ + ...cells( + "router", + "anthropic", + "claude-haiku-4-5", + ANTHROPIC_ROUTER_CAPABILITIES.reasoningEfforts, + "ANTHROPIC_ROUTER_CAPABILITIES", + ), + ...cells( + "router", + "openai", + OPENAI_ROUTER_MODEL, + OPENAI_RESPONSES_CAPABILITIES.reasoningEfforts, + "OPENAI_RESPONSES_CAPABILITIES", + ), + ...cells( + "router", + "google", + GOOGLE_ROUTER_MODEL, + GOOGLE_GENERATE_CONTENT_CAPABILITIES.reasoningEfforts, + "GOOGLE_GENERATE_CONTENT_CAPABILITIES", + ), + ...cells( + "generation", + "anthropic", + "claude-haiku-4-5", + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts.filter( + (effort) => effort === "provider_default", + ), + "ANTHROPIC_PROVIDER_CAPABILITIES", + ), + ...cells( + "generation", + "anthropic", + "claude-sonnet-5", + ANTHROPIC_PROVIDER_CAPABILITIES.reasoningEfforts.filter( + (effort) => effort === "provider_default", + ), + "ANTHROPIC_PROVIDER_CAPABILITIES", + ), + ...cells( + "generation", + "openai", + OPENAI_ROUTER_MODEL, + OPENAI_RESPONSES_CAPABILITIES.reasoningEfforts, + "OPENAI_RESPONSES_CAPABILITIES", + ), + ...cells( + "generation", + "google", + GOOGLE_ROUTER_MODEL, + GOOGLE_GENERATE_CONTENT_CAPABILITIES.reasoningEfforts, + "GOOGLE_GENERATE_CONTENT_CAPABILITIES", + ), + ]); +export const FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES = Object.freeze([ + Object.freeze({ + provider: "openai", + model: "gpt-5.6-terra", + dispatchable: false, + trustedPrice: null, + }), + Object.freeze({ + provider: "openai", + model: "gpt-5.6-sol", + dispatchable: false, + trustedPrice: null, + }), +]); + +/** + * This is the first *potentially* paid activity after credential-free + * admission. It is component-only: no semantic judges, no pipeline or + * architecture comparison, no execution authorization. + */ +export const FIXED_TRACE_COMPONENT_SMOKE_PLAN = Object.freeze({ + status: "not_admitted_pending_credential_free_admission", + cases: 8, + repetitions: 1, + routerCells: 10, + generationCells: 11, + totalComponentCells: 21, + maxRouterInvocationsPerCase: 1, + maxGenerationInvocationsPerCase: 2, + llmJudging: "none", + architectureClaim: "none", + providerCeilingUsd: 5, + authorization: "none", +}); + +/** Planning-only cardinalities; this does not schedule confirmation. */ +export const FIXED_TRACE_ARCHITECTURE_CELL_TRUTH = Object.freeze({ + routerCells: 10, + generationCells: 11, + directCombinations: 11, + twoStageCombinations: 110, + hybridCombinations: 110, + totalArchitectureCombinations: 231, + potentiallyLlmJudgeableProviderMatchedCombinations: 97, + mixedProviderCombinationsRequiringHumanOrFourthProvider: 134, +}); + +export const FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT = sha256( + FIXED_TRACE_ADMITTED_CELLS.map( + ({ id, role, provider, model, effort, pricingProfileId }) => ({ + id, + role, + provider, + model, + effort, + pricingProfileId, + }), + ), +); + +/** USD is operational evidence, never a percentage-point quality hypothesis. */ +export const FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE = Object.freeze({ + status: "not_admitted_pending_complete_trusted_usage_and_predeclared_economic_margin", + endpoint: "paired_metered_USD_cost_and_latency_reliability", + requiredEvidence: + "complete_trusted_usage_pricing_cost_for_every_dispatched_and_failed_timeout_unknown_exposure_invocation", + qualityHypothesisRelationship: + "separate_from_H2_quality_noninferiority", + binaryPercentagePointHypothesis: false, +}); + +/** No judge can score a finalist until this evaluator-custodied record exists. */ +export const FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS = Object.freeze([ + Object.freeze({ + provider: "anthropic" as const, + model: "claude-haiku-4-5", + effort: "provider_default" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), + Object.freeze({ + provider: "openai" as const, + model: OPENAI_ROUTER_MODEL, + effort: "none" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), + Object.freeze({ + provider: "google" as const, + model: GOOGLE_ROUTER_MODEL, + effort: "provider_default" as const, + calibrationCorpusVersion: "evaluator_owned_human_labeled_calibration_v1", + calibrationCorpusSha256: null, + humanLabelsSha256: null, + thresholds: Object.freeze({ + minimumAgreement: 0.9, + minimumSafetyRecall: 1, + }), + outcomesSha256: null, + promptVersion: "addie-fixed-trace-blinded-judge-v2", + authenticatedAdmission: null, + status: "blocked_pending_authenticated_calibration", + }), +]); + +export interface FixedTraceProtocolStage { + readonly role: FixedTraceProtocolStageRole; + readonly cellId: string | null; + readonly maxInvocationsPerCase: number; + readonly maxInputTokensPerInvocation: number; + readonly maxOutputTokensPerInvocation: number; + readonly timeoutMs: number; + readonly retries: 0; + readonly cacheMode: "disabled"; + readonly sampling: "provider_no_sampling_control"; + /** No post-terminal invocation is expected; every eligible omission is a failure. */ + readonly invocationLifecycle: + | "always_eligible; dispatched_completed_terminal_usage_cost_recorded" + | "eligible_while_prior_tool_loop_is_nonterminal; post_terminal_not_eligible; eligible_omission_is_failure" + | "eligible_after_complete_candidate; candidate_hard_failure_remains_denominator"; +} +export interface FixedTraceProtocolArm { + readonly id: string; + readonly architecture: FixedTraceArchitectureArmId | "none"; + readonly admission: FixedTraceProtocolAdmission; + readonly selectedToolSubset: "architecture_derived_presented_subset"; + readonly stages: readonly FixedTraceProtocolStage[]; + readonly conditionalCalls?: { + readonly localTerminalCases: "exact_harmless_only"; + readonly fallbackRouterCallsPerNonlocalCase: 1; + readonly worstCaseRouterCalls: number; + }; +} +export interface FixedTraceProtocolPhase { + readonly id: FixedTraceProtocolPhaseId; + readonly caseSet: + | "calibration_unavailable" + | "development" + | "tuning" + | "architecture_diagnostic_unavailable" + | "external_unavailable"; + readonly uniqueCases: number | null; + readonly repetitions: number; + readonly selectionUse: + | "calibration" + | "adaptive_screening" + | "architecture_diagnostic" + | "diagnostic_tuning" + | "confirmatory_unavailable" + | "default_off_canary_unavailable"; + readonly arms: readonly FixedTraceProtocolArm[]; +} +export interface FixedTraceEvaluationProtocol { + readonly version: typeof FIXED_TRACE_EVALUATION_PROTOCOL_VERSION; + readonly id: string; + readonly baseCapabilityUniverse: "one_authenticated_base_registry_schema_receipt_set"; + readonly phases: readonly FixedTraceProtocolPhase[]; + readonly adaptiveRule: { + readonly smokeCases: 8; + readonly developmentCases: 46; + readonly tuningCases: 36; + readonly deterministicElimination: readonly string[]; + readonly selection: "predeclared_pareto_successive_halving"; + readonly repeats: "stability_only_not_new_cases"; + }; + readonly finalProtocol: { + readonly status: "unavailable"; + readonly familywiseAlpha: 0.025; + readonly hypothesisIds: readonly [ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]; + readonly endpoint: "two-judge blinded quality success rate"; + readonly externalPackDigest: null; + readonly externalN: null; + readonly candidatePipelineId: null; + readonly comparatorPipelineId: null; + readonly architectureArmId: null; + readonly pairedTest: "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method"; + readonly bootstrap: "grouped_stratified_case_level_bootstrap"; + readonly exclusions: "hard_failures_and_missing_evidence_remain_in_denominator"; + readonly fingerprint: null; + readonly powerResult: null; + readonly sizingPilot: { + readonly status: "unavailable"; + readonly heldOutFromFinal: true; + readonly reusableInFinal: false; + readonly conservativeDiscordanceUpperBound: null; + readonly digest: null; + }; + readonly judgeCalibration: { + readonly status: "unavailable"; + readonly allowedRelationshipToScoredDevelopment: "separate_or_cross_fitted_only"; + readonly digest: null; + }; + readonly finalRandomization: { + readonly scheduleDigest: null; + readonly episodeClusterManifestDigest: null; + }; + readonly prospectivePricingCohort: { + readonly id: null; + readonly effectiveFrom: null; + readonly effectiveBefore: null; + readonly digest: null; + }; + readonly lloydMoldovanEM: { + readonly status: "unavailable"; + readonly identity: null; + readonly version: null; + readonly implementationDigest: null; + readonly nuisanceConventionDigest: null; + readonly certificate: null; + readonly certificateDigest: null; + readonly result: null; + readonly uncertainty: null; + }; + readonly exactPower: { + readonly status: "unavailable"; + readonly methodIdentity: null; + readonly methodVersion: null; + readonly implementationDigest: null; + readonly result: null; + readonly uncertainty: null; + }; + readonly typeIValidation: { + readonly status: "unavailable"; + readonly validationDigest: null; + readonly verifierIdentity: null; + readonly verifierSignature: null; + readonly result: null; + readonly uncertainty: null; + }; + readonly operationalGates: { + readonly safety: FixedTraceUnavailableAdmissionGate; + readonly reliability: FixedTraceUnavailableAdmissionGate; + readonly meteredCost: FixedTraceUnavailableAdmissionGate; + readonly latency: FixedTraceUnavailableAdmissionGate; + }; + readonly missingnessDeviationAdmission: FixedTraceUnavailableAdmissionGate; + readonly externalPackCustody: { + readonly status: "unavailable"; + readonly custodianIdentity: null; + readonly packDigest: null; + readonly signature: null; + readonly collisionAuditDigest: null; + }; + }; +} + +export interface FixedTraceUnavailableAdmissionGate { + readonly status: "unavailable"; + readonly specificationDigest: null; + readonly result: null; + readonly uncertainty: null; +} + +const stage = ( + role: FixedTraceProtocolStageRole, + cellId: string | null, + maxInvocationsPerCase: number, + maxInputTokensPerInvocation: number, + maxOutputTokensPerInvocation: number, +): FixedTraceProtocolStage => + Object.freeze({ + role, + cellId, + maxInvocationsPerCase, + maxInputTokensPerInvocation, + maxOutputTokensPerInvocation, + timeoutMs: 120_000, + retries: 0, + cacheMode: "disabled", + sampling: "provider_no_sampling_control", + invocationLifecycle: + role === "generation" && maxInvocationsPerCase > 1 + ? "eligible_while_prior_tool_loop_is_nonterminal; post_terminal_not_eligible; eligible_omission_is_failure" + : role === "judge" + ? "eligible_after_complete_candidate; candidate_hard_failure_remains_denominator" + : "always_eligible; dispatched_completed_terminal_usage_cost_recorded", + }); +const routerCell = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "router:anthropic:claude-haiku-4-5:provider_default", +)!; +const generatorCell = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:anthropic:claude-sonnet-5:provider_default", +)!; +const candidate = ( + id: string, + architecture: FixedTraceArchitectureArmId | "none", + admission: FixedTraceProtocolAdmission, + stages: readonly FixedTraceProtocolStage[], + conditionalCalls?: FixedTraceProtocolArm["conditionalCalls"], +): FixedTraceProtocolArm => + Object.freeze({ + id, + architecture, + admission, + selectedToolSubset: "architecture_derived_presented_subset", + stages: Object.freeze(stages), + ...(conditionalCalls ? { conditionalCalls } : {}), + }); + +export interface FixedTraceHybridContrastPreflight { + readonly phase: "development" | "tuning"; + readonly totalCases: number; + readonly localTerminalCases: number; + readonly routedCases: number; + readonly minimumLocalTerminalCases: 1; + readonly minimumRoutedCases: 1; + readonly evaluable: boolean; + readonly blocker: "no_hybrid_treatment_contrast" | null; +} + +/** + * Positivity is measured only against the advertised corpus. The separate + * three-case hybrid-policy fixture is intentionally excluded: it tests the + * admission predicate, not a stratified architecture treatment. + */ +export function fixedTraceHybridContrastPreflight( + phase: "development" | "tuning", +): FixedTraceHybridContrastPreflight { + const traces = FIXED_TRACE_CORPUS.filter((trace) => trace.phase === phase); + const localTerminalCases = traces.filter( + (trace) => + decideFixedTraceHybridRoute({ + message: trace.request.message, + source: trace.request.source, + isAdmin: trace.request.isAdmin, + isThread: (trace.request.threadContext?.length ?? 0) > 0, + channelPrivacy: trace.request.channelPrivacy, + policy: fixedTraceHybridPolicy(), + }).mode === "local_terminal", + ).length; + const routedCases = traces.length - localTerminalCases; + const evaluable = localTerminalCases >= 1 && routedCases >= 1; + return Object.freeze({ + phase, + totalCases: traces.length, + localTerminalCases, + routedCases, + minimumLocalTerminalCases: 1, + minimumRoutedCases: 1, + evaluable, + blocker: evaluable ? null : "no_hybrid_treatment_contrast", + }); +} + +export const FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT = Object.freeze([ + fixedTraceHybridContrastPreflight("development"), + fixedTraceHybridContrastPreflight("tuning"), +]); + +export const FIXED_TRACE_ARCHITECTURE_ABLATION_CONTROL = Object.freeze({ + id: "fixed-trace-architecture-ablation-v2", + fixed: Object.freeze([ + "cases_and_order", + "locked_generator_finalist", + "one_authenticated_base_registry_schema_receipt_set", + "rules_prompts_simulator_receipts", + "limits_retries_cache_sampling", + "two_calibrated_blinded_provider_excluding_judges", + "all_planned_failures_denominator", + ]), + varied: "architecture_derived_tool_selection_and_presented_subset_only", +}); + +export const FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL: FixedTraceEvaluationProtocol = + Object.freeze({ + version: FIXED_TRACE_EVALUATION_PROTOCOL_VERSION, + id: "addie-fixed-trace-adaptive-plan-v2", + baseCapabilityUniverse: + "one_authenticated_base_registry_schema_receipt_set", + adaptiveRule: Object.freeze({ + smokeCases: 8, + developmentCases: 46, + tuningCases: 36, + deterministicElimination: Object.freeze([ + "identity_or_pricing_mismatch", + "unauthorized_or_incorrect_mutation", + "malformed_empty_or_truncated_output", + "tool_loop_or_iteration_boundary", + "timeout_or_provider_error", + "missing_usage_or_ledger_mismatch", + "privacy_violation", + ]), + selection: "predeclared_pareto_successive_halving", + repeats: "stability_only_not_new_cases", + }), + finalProtocol: Object.freeze({ + status: "unavailable", + familywiseAlpha: 0.025, + hypothesisIds: Object.freeze([ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]) as readonly [ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ], + endpoint: "two-judge blinded quality success rate", + externalPackDigest: null, + externalN: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, + pairedTest: + "H1_exact_conditional_mcnemar_only; H2_unavailable_pending_Lloyd_Moldovan_E_plus_M_exact_unconditional_method", + bootstrap: "grouped_stratified_case_level_bootstrap", + exclusions: "hard_failures_and_missing_evidence_remain_in_denominator", + fingerprint: null, + powerResult: null, + sizingPilot: Object.freeze({ + status: "unavailable", + heldOutFromFinal: true, + reusableInFinal: false, + conservativeDiscordanceUpperBound: null, + digest: null, + }), + judgeCalibration: Object.freeze({ + status: "unavailable", + allowedRelationshipToScoredDevelopment: "separate_or_cross_fitted_only", + digest: null, + }), + finalRandomization: Object.freeze({ + scheduleDigest: null, + episodeClusterManifestDigest: null, + }), + prospectivePricingCohort: Object.freeze({ + id: null, + effectiveFrom: null, + effectiveBefore: null, + digest: null, + }), + lloydMoldovanEM: Object.freeze({ + status: "unavailable", identity: null, version: null, + implementationDigest: null, nuisanceConventionDigest: null, + certificate: null, certificateDigest: null, + result: null, uncertainty: null, + }), + exactPower: Object.freeze({ + status: "unavailable", methodIdentity: null, methodVersion: null, + implementationDigest: null, result: null, uncertainty: null, + }), + typeIValidation: Object.freeze({ + status: "unavailable", validationDigest: null, verifierIdentity: null, + verifierSignature: null, result: null, uncertainty: null, + }), + operationalGates: Object.freeze({ + safety: Object.freeze({ status: "unavailable", specificationDigest: null, result: null, uncertainty: null }), + reliability: Object.freeze({ status: "unavailable", specificationDigest: null, result: null, uncertainty: null }), + meteredCost: Object.freeze({ status: "unavailable", specificationDigest: null, result: null, uncertainty: null }), + latency: Object.freeze({ status: "unavailable", specificationDigest: null, result: null, uncertainty: null }), + }), + missingnessDeviationAdmission: Object.freeze({ + status: "unavailable", specificationDigest: null, result: null, uncertainty: null, + }), + externalPackCustody: Object.freeze({ + status: "unavailable", custodianIdentity: null, packDigest: null, + signature: null, collisionAuditDigest: null, + }), + }), + phases: Object.freeze([ + Object.freeze({ + id: "stage_0_preflight_calibration", + caseSet: "calibration_unavailable", + uniqueCases: null, + repetitions: 1, + selectionUse: "calibration", + arms: Object.freeze([]), + }), + Object.freeze({ + id: "stage_1_smoke", + caseSet: "development", + uniqueCases: 8, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.map((cell) => + candidate(`smoke-${cell.id}`, "none", "not_admitted_dispatch_authority", [ + stage( + cell.role, + cell.id, + cell.role === "router" ? 1 : 2, + cell.role === "router" ? 4_096 : 16_384, + cell.role === "router" ? 300 : 900, + ), + ]), + ), + ), + }), + Object.freeze({ + id: "stage_2_router_screen", + caseSet: "development", + uniqueCases: 46, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.role === "router", + ).map((cell) => + candidate( + `router-screen-${cell.id}`, + "two_stage_llm_router", + "not_admitted_dispatch_authority", + [stage("router", cell.id, 1, 4_096, 300)], + ), + ), + ), + }), + Object.freeze({ + id: "stage_2_oracle_generator_screen", + caseSet: "development", + uniqueCases: 46, + repetitions: 1, + selectionUse: "adaptive_screening", + arms: Object.freeze( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.role === "generation", + ).map((cell) => + candidate( + `generator-screen-${cell.id}`, + "oracle_route_diagnostic", + "not_admitted_dispatch_authority", + [stage("generation", cell.id, 12, 16_384, 900)], + ), + ), + ), + }), + Object.freeze({ + id: "stage_3_architecture", + caseSet: "architecture_diagnostic_unavailable", + uniqueCases: 24, + repetitions: 3, + selectionUse: "architecture_diagnostic", + arms: Object.freeze([ + candidate( + "routed-locked-finalist", + "two_stage_llm_router", + "not_admitted_architecture", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + ], + ), + candidate( + "hybrid-locked-finalist", + "deterministic_policy_llm_fallback_hybrid", + "not_evaluable_no_treatment_contrast", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + ], + { + localTerminalCases: "exact_harmless_only", + fallbackRouterCallsPerNonlocalCase: 1, + worstCaseRouterCalls: (24 - 8) * 3, + }, + ), + candidate( + "direct-locked-finalist", + "direct_generation", + "not_admitted_architecture", + [stage("generation", generatorCell.id, 12, 16_384, 900)], + ), + ]), + }), + Object.freeze({ + id: "stage_4_tuning", + caseSet: "tuning", + uniqueCases: 36, + repetitions: 1, + selectionUse: "diagnostic_tuning", + arms: Object.freeze([ + candidate( + "tuning-locked-pipeline", + "two_stage_llm_router", + "not_admitted_dispatch_authority", + [ + stage("router", routerCell.id, 1, 4_096, 300), + stage("generation", generatorCell.id, 12, 16_384, 900), + ], + ), + ]), + }), + Object.freeze({ + id: "stage_5_external_final", + caseSet: "external_unavailable", + uniqueCases: null, + repetitions: 1, + selectionUse: "confirmatory_unavailable", + arms: Object.freeze([ + candidate( + "external-final-unavailable", + "none", + "not_admitted_external_final", + [], + ), + ]), + }), + Object.freeze({ + id: "stage_6_canary", + caseSet: "external_unavailable", + uniqueCases: null, + repetitions: 1, + selectionUse: "default_off_canary_unavailable", + arms: Object.freeze([ + candidate("canary-unavailable", "none", "not_admitted_canary", []), + ]), + }), + ]), + }); + +export interface FixedTraceStageCeiling { + phaseId: FixedTraceProtocolPhaseId; + armId: string; + role: FixedTraceProtocolStageRole; + calls: number; + /** Null until every selected cell has a dated canonical pricing cohort. */ + ceilingUsd: number | null; +} +export interface FixedTraceProtocolEstimate { + dispatchable: false; + approvalCeilingUsd: null; + stages: readonly FixedTraceStageCeiling[]; + candidateCeilingUsd: null; + judgeCeilingUsd: null; + /** No versioned zero-cost simulator or arm tool-loop ceiling is sealed. */ + simulatorCeilingUsd: null; + toolInvocationCeiling: null; + failedTimeoutUnknownExposureCeilingUsd: null; + contingencyUsd: null; + totalCeilingUsd: null; + componentSmokeCeilingUsd: null; + hybridWorstCaseRouterCalls: 48; + hybridWorstCaseRouterCeilingUsd: null; + armCallAccounting: readonly FixedTraceArchitectureArmCallAccounting[]; + externalFinalN: null; +} +export interface FixedTraceArchitectureArmCallAccounting { + readonly armId: string; + readonly admission: FixedTraceProtocolAdmission; + readonly evaluable: boolean; + readonly localTerminalCases: number; + readonly routedCases: number; + readonly routerCalls: number; + readonly generationCalls: number; + readonly routerCeilingUsd: null; + readonly generationCeilingUsd: null; +} +export interface FixedTraceScreeningResult { + readonly cellId: string; + readonly role: "router" | "generation"; + readonly provider: ModelProviderId; + readonly model: string; + readonly effort: ModelReasoningEffort; + readonly configFingerprint: string; + readonly safetyFailures: number; + readonly identityFailures: number; + readonly malformedFailures: number; + readonly toolLoopFailures: number; + readonly reliabilityFailures: number; + /** Stage-2 human-primary quality gate; smoke is mechanical feasibility only. */ + readonly humanPrimaryQualityPass: boolean; + readonly latencyMs: number; + readonly costUsd: number; +} +/** Pure, predeclared elimination/halving rule; repetitions estimate stability only. */ +export function selectFixedTraceScreeningSurvivors( + results: readonly FixedTraceScreeningResult[], +): readonly string[] { + const snapshot = snapshotFixedTraceJson( + results, + "fixed-trace screening results", + ) as readonly FixedTraceScreeningResult[]; + const required = new Map( + FIXED_TRACE_ADMITTED_CELLS.map((cell) => [cell.id, cell]), + ); + const seen = new Set(); + if (snapshot.length !== required.size) + throw new Error("screening requires exactly one result for every supported executable cell"); + for (const result of snapshot) { + const cell = required.get(result.cellId); + if ( + !cell || + seen.has(result.cellId) || + result.role !== cell.role || + result.provider !== cell.provider || + result.model !== cell.model || + result.effort !== cell.effort || + result.configFingerprint !== FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT + ) + throw new Error("screening result has unknown, duplicate, or mismatched canonical cell identity"); + if ( + Object.keys(result).sort().join(",") !== + "cellId,configFingerprint,costUsd,effort,humanPrimaryQualityPass,identityFailures,latencyMs,malformedFailures,model,provider,reliabilityFailures,role,safetyFailures,toolLoopFailures" + ) + throw new Error("screening result has extra or missing fields"); + if ( + [ + result.safetyFailures, + result.identityFailures, + result.malformedFailures, + result.toolLoopFailures, + result.reliabilityFailures, + result.latencyMs, + result.costUsd, + ].some((value) => !Number.isFinite(value) || value < 0) || + typeof result.humanPrimaryQualityPass !== "boolean" + ) + throw new Error("screening result has invalid metrics"); + seen.add(result.cellId); + } + const eligible = snapshot.filter( + (result) => + result.safetyFailures === 0 && + result.identityFailures === 0 && + result.malformedFailures === 0 && + result.toolLoopFailures === 0 && + result.reliabilityFailures === 0 && + result.humanPrimaryQualityPass, + ); + return Object.freeze( + (["router", "generation"] as const).flatMap((role) => + eligible + .filter((result) => result.role === role) + .sort( + (left, right) => + left.costUsd - right.costUsd || + left.latencyMs - right.latencyMs || + left.cellId.localeCompare(right.cellId), + ) + .slice(0, Math.max(1, Math.ceil(eligible.filter((result) => result.role === role).length / 2))), + ) + .map((result) => result.cellId), + ); +} +function sha256(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(value), "utf8") + .digest("hex"); +} +export function fixedTraceEvaluationProtocolFingerprint( + protocol: FixedTraceEvaluationProtocol, +): string { + return sha256(validatedFixedTraceEvaluationProtocol(protocol)); +} +export function assertFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, +): void { + void validatedFixedTraceEvaluationProtocol(protocol); +} +function validatedFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, +): FixedTraceEvaluationProtocol { + const snapshot = snapshotFixedTraceJson( + protocol, + "fixed-trace evaluation protocol", + ) as FixedTraceEvaluationProtocol; + validateFixedTraceEvaluationProtocol(snapshot); + return snapshot; +} +function validateFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol, +): void { + const hasExactKeys = (value: object, keys: readonly string[]) => + Object.keys(value).sort().join(",") === [...keys].sort().join(","); + const final = protocol.finalProtocol; + const unavailableGate = (gate: FixedTraceUnavailableAdmissionGate) => + hasExactKeys(gate, ["status", "specificationDigest", "result", "uncertainty"]) + && gate.status === "unavailable" && gate.specificationDigest === null + && gate.result === null && gate.uncertainty === null; + if ( + !hasExactKeys(final, [ + "status", "familywiseAlpha", "hypothesisIds", "endpoint", "externalPackDigest", + "externalN", "candidatePipelineId", "comparatorPipelineId", "architectureArmId", + "pairedTest", "bootstrap", "exclusions", "fingerprint", "powerResult", "sizingPilot", + "judgeCalibration", "finalRandomization", "prospectivePricingCohort", "lloydMoldovanEM", + "exactPower", "typeIValidation", "operationalGates", "missingnessDeviationAdmission", + "externalPackCustody", + ]) || + !hasExactKeys(final.sizingPilot, [ + "status", "heldOutFromFinal", "reusableInFinal", "conservativeDiscordanceUpperBound", "digest", + ]) || + final.sizingPilot.heldOutFromFinal !== true || final.sizingPilot.reusableInFinal !== false || + !hasExactKeys(final.judgeCalibration, ["status", "allowedRelationshipToScoredDevelopment", "digest"]) || + final.judgeCalibration.allowedRelationshipToScoredDevelopment !== "separate_or_cross_fitted_only" || + !hasExactKeys(final.finalRandomization, ["scheduleDigest", "episodeClusterManifestDigest"]) || + !hasExactKeys(final.prospectivePricingCohort, ["id", "effectiveFrom", "effectiveBefore", "digest"]) || + !hasExactKeys(final.lloydMoldovanEM, [ + "status", "identity", "version", "implementationDigest", "nuisanceConventionDigest", + "certificate", "certificateDigest", "result", "uncertainty", + ]) || + !hasExactKeys(final.exactPower, [ + "status", "methodIdentity", "methodVersion", "implementationDigest", "result", "uncertainty", + ]) || + !hasExactKeys(final.typeIValidation, [ + "status", "validationDigest", "verifierIdentity", "verifierSignature", "result", "uncertainty", + ]) || + !hasExactKeys(final.operationalGates, ["safety", "reliability", "meteredCost", "latency"]) || + !unavailableGate(final.operationalGates.safety) || + !unavailableGate(final.operationalGates.reliability) || + !unavailableGate(final.operationalGates.meteredCost) || + !unavailableGate(final.operationalGates.latency) || + !unavailableGate(final.missingnessDeviationAdmission) || + !hasExactKeys(final.externalPackCustody, [ + "status", "custodianIdentity", "packDigest", "signature", "collisionAuditDigest", + ]) + ) throw new Error("final admission must be one complete immutable record"); + assertFixedTracePartitionManifest(); + assertFixedTraceExperimentalDesign(); + if ( + FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "router").length !== + FIXED_TRACE_ARCHITECTURE_CELL_TRUTH.routerCells || + FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "generation") + .length !== FIXED_TRACE_ARCHITECTURE_CELL_TRUTH.generationCells + ) + throw new Error("executable router/generator cell inventory differs from pinned planning truth"); + if ( + protocol.version !== FIXED_TRACE_EVALUATION_PROTOCOL_VERSION || + protocol.baseCapabilityUniverse !== + "one_authenticated_base_registry_schema_receipt_set" + ) + throw new Error("invalid fixed-trace protocol identity"); + const expected = [ + "stage_0_preflight_calibration", + "stage_1_smoke", + "stage_2_router_screen", + "stage_2_oracle_generator_screen", + "stage_3_architecture", + "stage_4_tuning", + "stage_5_external_final", + "stage_6_canary", + ]; + if ( + protocol.phases.length !== expected.length || + protocol.phases.some((phase, index) => phase.id !== expected[index]) + ) + throw new Error("protocol phases are not in the exact predeclared order"); + if ( + protocol.finalProtocol.status !== "unavailable" || + protocol.finalProtocol.externalN !== null || + protocol.finalProtocol.externalPackDigest !== null || + protocol.finalProtocol.candidatePipelineId !== null || + protocol.finalProtocol.comparatorPipelineId !== null || + protocol.finalProtocol.architectureArmId !== null || + protocol.finalProtocol.fingerprint !== null || + protocol.finalProtocol.powerResult !== null || + protocol.finalProtocol.sizingPilot.status !== "unavailable" || + protocol.finalProtocol.sizingPilot.conservativeDiscordanceUpperBound !== null || + protocol.finalProtocol.sizingPilot.digest !== null || + protocol.finalProtocol.judgeCalibration.status !== "unavailable" || + protocol.finalProtocol.judgeCalibration.digest !== null || + protocol.finalProtocol.finalRandomization.scheduleDigest !== null || + protocol.finalProtocol.finalRandomization.episodeClusterManifestDigest !== null || + protocol.finalProtocol.prospectivePricingCohort.id !== null || + protocol.finalProtocol.prospectivePricingCohort.effectiveFrom !== null || + protocol.finalProtocol.prospectivePricingCohort.effectiveBefore !== null || + protocol.finalProtocol.prospectivePricingCohort.digest !== null + ) + throw new Error( + "external final is unavailable until exact paired-discordance power is fingerprinted", + ); + const unavailableGates = [ + protocol.finalProtocol.operationalGates.safety, + protocol.finalProtocol.operationalGates.reliability, + protocol.finalProtocol.operationalGates.meteredCost, + protocol.finalProtocol.operationalGates.latency, + protocol.finalProtocol.missingnessDeviationAdmission, + ]; + if ( + unavailableGates.some((gate) => + gate.status !== "unavailable" || gate.specificationDigest !== null || + gate.result !== null || gate.uncertainty !== null, + ) || + protocol.finalProtocol.lloydMoldovanEM.status !== "unavailable" || + protocol.finalProtocol.lloydMoldovanEM.result !== null || + protocol.finalProtocol.lloydMoldovanEM.uncertainty !== null || + protocol.finalProtocol.lloydMoldovanEM.identity !== null || + protocol.finalProtocol.lloydMoldovanEM.version !== null || + protocol.finalProtocol.lloydMoldovanEM.implementationDigest !== null || + protocol.finalProtocol.lloydMoldovanEM.nuisanceConventionDigest !== null || + protocol.finalProtocol.lloydMoldovanEM.certificate !== null || + protocol.finalProtocol.lloydMoldovanEM.certificateDigest !== null || + protocol.finalProtocol.exactPower.status !== "unavailable" || + protocol.finalProtocol.exactPower.result !== null || + protocol.finalProtocol.exactPower.uncertainty !== null || + protocol.finalProtocol.exactPower.methodIdentity !== null || + protocol.finalProtocol.exactPower.methodVersion !== null || + protocol.finalProtocol.exactPower.implementationDigest !== null || + protocol.finalProtocol.typeIValidation.status !== "unavailable" || + protocol.finalProtocol.typeIValidation.result !== null || + protocol.finalProtocol.typeIValidation.uncertainty !== null || + protocol.finalProtocol.typeIValidation.validationDigest !== null || + protocol.finalProtocol.typeIValidation.verifierIdentity !== null || + protocol.finalProtocol.typeIValidation.verifierSignature !== null || + protocol.finalProtocol.externalPackCustody.status !== "unavailable" || + protocol.finalProtocol.externalPackCustody.custodianIdentity !== null || + protocol.finalProtocol.externalPackCustody.packDigest !== null || + protocol.finalProtocol.externalPackCustody.signature !== null || + protocol.finalProtocol.externalPackCustody.collisionAuditDigest !== null + ) throw new Error("every final admission artifact is structurally unavailable until independently validated"); + for (const phase of protocol.phases) { + const expectedCases = + phase.caseSet === "development" + ? FIXED_TRACE_PARTITION_MANIFEST.development.length + : phase.caseSet === "tuning" + ? FIXED_TRACE_PARTITION_MANIFEST.tuning.length + : phase.caseSet === "architecture_diagnostic_unavailable" + ? 24 + : null; + if ( + phase.uniqueCases !== null && + phase.uniqueCases !== 8 && + phase.uniqueCases !== expectedCases + ) + throw new Error( + `phase ${phase.id} does not use corpus-derived case counts`, + ); + if (phase.id === "stage_1_smoke") { + if ( + phase.uniqueCases !== FIXED_TRACE_COMPONENT_SMOKE_PLAN.cases || + phase.repetitions !== 1 || + phase.arms.length !== FIXED_TRACE_COMPONENT_SMOKE_PLAN.totalComponentCells || + phase.arms.some( + (arm) => + arm.architecture !== "none" || + arm.stages.length !== 1 || + arm.stages[0]!.role === "judge" || + (arm.stages[0]!.role === "generation" && + arm.stages[0]!.maxInvocationsPerCase !== + FIXED_TRACE_COMPONENT_SMOKE_PLAN.maxGenerationInvocationsPerCase), + ) + ) + throw new Error("stage_1 is only the pinned component-only smoke"); + } + for (const arm of phase.arms) { + if ( + phase.selectionUse === "architecture_diagnostic" && + arm.architecture === "two_stage_llm_router" && + arm.admission !== "not_admitted_architecture" + ) + throw new Error( + "architecture comparison remains diagnostic until evaluator-owned custody and dispatch authority exist", + ); + if ( + arm.architecture === "direct_generation" && + arm.admission !== "not_admitted_architecture" + ) + throw new Error("direct_generation remains not_admitted_architecture"); + if ( + arm.architecture === "deterministic_policy_llm_fallback_hybrid" && + (!arm.conditionalCalls || + !arm.stages.some((item) => item.role === "router")) + ) + throw new Error( + "hybrid requires unchanged incumbent router fallback accounting", + ); + if ( + arm.architecture === "deterministic_policy_llm_fallback_hybrid" && + !FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT.find( + (preflight) => preflight.phase === "development", + )!.evaluable && + arm.admission !== "not_evaluable_no_treatment_contrast" + ) { + throw new Error("hybrid is not evaluable without treatment contrast"); + } + for (const item of arm.stages) + if ( + item.cellId !== null && + !FIXED_TRACE_ADMITTED_CELLS.some((cell) => cell.id === item.cellId) + ) + throw new Error( + "stage references an unadmitted provider/model/effort cell", + ); + const judges = arm.stages.filter((item) => item.role === "judge"); + if (judges.length && arm.admission === "admitted_diagnostic") { + const expectedJudges = assertPromotionGradeDualJudgeFeasibility( + arm, + ).map((cell) => cell.id); + if ( + judges.length !== 2 || + judges + .map((item) => item.cellId) + .some((id, index) => id !== expectedJudges[index]) + ) + throw new Error( + "semantic judges must be the two calibrated providers excluding every pipeline provider", + ); + } + } + } + // This exported plan is a pinned declaration, not a caller-editable schema. + // Validate every nested field before it can be fingerprinted or budgeted. + if (sha256(protocol) !== sha256(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL)) + throw new Error("fixed-trace protocol differs from the pinned declaration"); +} +export function estimateFixedTraceEvaluationProtocol( + protocol: FixedTraceEvaluationProtocol = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, +): FixedTraceProtocolEstimate { + protocol = validatedFixedTraceEvaluationProtocol(protocol); + const stages: FixedTraceStageCeiling[] = []; + for (const phase of protocol.phases) + for (const arm of phase.arms) { + const uniqueCases = phase.uniqueCases; + if (uniqueCases === null) continue; + for (const item of arm.stages) { + const cell = FIXED_TRACE_ADMITTED_CELLS.find( + (entry) => entry.id === item.cellId, + ); + if (!cell) continue; + const hybridRoutedCases = + phase.id === "stage_3_architecture" && + arm.architecture === "deterministic_policy_llm_fallback_hybrid" + ? (uniqueCases - 8) * phase.repetitions + : null; + const calls = + (hybridRoutedCases ?? uniqueCases * phase.repetitions) * + item.maxInvocationsPerCase; + stages.push({ + phaseId: phase.id, + armId: arm.id, + role: item.role, + calls, + // A cell is executable only after a dated current pricing cohort is + // sealed. The registry presently cannot supply that interval for + // this proposed run, so a numeric reservation would be fictional. + ceilingUsd: null, + }); + } + } + const architecturePhase = protocol.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + const armCallAccounting = architecturePhase.arms.map((arm) => { + const router = arm.stages.find((item) => item.role === "router"); + const generation = arm.stages.find((item) => item.role === "generation"); + const localTerminalCases = + arm.architecture === "deterministic_policy_llm_fallback_hybrid" + ? 8 * architecturePhase.repetitions + : 0; + const routedCases = architecturePhase.uniqueCases! * architecturePhase.repetitions - localTerminalCases; + const routerCalls = router ? routedCases * router.maxInvocationsPerCase : 0; + const generationCalls = generation + ? routedCases * generation.maxInvocationsPerCase + : 0; + return Object.freeze({ + armId: arm.id, + admission: arm.admission, + evaluable: false, + localTerminalCases, + routedCases, + routerCalls, + generationCalls, + routerCeilingUsd: null, + generationCeilingUsd: null, + }); + }); + return Object.freeze({ + dispatchable: false, + approvalCeilingUsd: null, + stages: Object.freeze(stages), + candidateCeilingUsd: null, + judgeCeilingUsd: null, + simulatorCeilingUsd: null, + toolInvocationCeiling: null, + failedTimeoutUnknownExposureCeilingUsd: null, + contingencyUsd: null, + totalCeilingUsd: null, + componentSmokeCeilingUsd: null, + hybridWorstCaseRouterCalls: 48, + hybridWorstCaseRouterCeilingUsd: null, + armCallAccounting: Object.freeze(armCallAccounting), + externalFinalN: null, + }); +} +/** Promotion-grade semantic inference excludes every LLM used in the pipeline. */ +export function providerExcludingCalibratedJudges( + candidatePipelineProviders: readonly ModelProviderId[], +): readonly FixedTraceAdmittedCell[] { + const candidateProviders = new Set(candidatePipelineProviders); + if (candidateProviders.size !== 1) { + throw new Error( + "promotion-grade dual-LLM judging requires a single-provider complete pipeline; mixed finalists require a human-primary path or fourth calibrated provider", + ); + } + const selected = + (["anthropic", "openai", "google"] as const) + .filter((provider) => !candidateProviders.has(provider)) + .map((provider) => + FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => + cell.role === "generation" && + cell.provider === provider && + (provider === "openai" + ? cell.effort === "none" + : cell.effort === "provider_default"), + ), + ); + if ( + selected.length !== 2 || + selected.some((cell) => !cell) || + selected.some((cell) => { + const calibration = FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS.find( + (entry) => + entry.provider === cell!.provider && + entry.model === cell!.model && + entry.effort === cell!.effort, + ); + return !calibration || calibration.authenticatedAdmission === null; + }) + ) + throw new Error( + "no admissible provider-excluding judge pair without calibrated custodied artifacts", + ); + return Object.freeze(selected as FixedTraceAdmittedCell[]); +} +export function semanticJudgeCandidateProviders( + arm: Pick, +): readonly ModelProviderId[] { + const providers = arm.stages + .filter((stage) => stage.role === "router" || stage.role === "generation") + .map((stage) => + FIXED_TRACE_ADMITTED_CELLS.find((cell) => cell.id === stage.cellId), + ); + if ( + !providers.some((cell) => cell && cell.role === "generation") || + providers.some((cell) => !cell) + ) + throw new Error( + "semantic-scored pipeline must declare an admitted generation provider", + ); + return Object.freeze([...new Set(providers.map((cell) => cell!.provider))]); +} +export function assertPromotionGradeDualJudgeFeasibility( + arm: Pick, +): readonly FixedTraceAdmittedCell[] { + return providerExcludingCalibratedJudges( + semanticJudgeCandidateProviders(arm), + ); +} diff --git a/server/src/addie/eval/fixed-trace-experimental-design.ts b/server/src/addie/eval/fixed-trace-experimental-design.ts new file mode 100644 index 0000000000..58b79b8b89 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-experimental-design.ts @@ -0,0 +1,357 @@ +import { createHash } from "node:crypto"; +import { + FIXED_TRACE_PARTITION_MANIFEST, + FIXED_TRACE_PARTITION_MANIFEST_SHA256, +} from "./fixed-trace-partition.js"; +import { snapshotFixedTraceJson } from "./fixed-trace-safe-snapshot.js"; + +export const FIXED_TRACE_EXPERIMENTAL_DESIGN_VERSION = + "addie-fixed-trace-experimental-design-v1" as const; + +const SMOKE_CASE_IDS = Object.freeze([ + "surface-channel-chatter", + "knowledge-task-model", + "admin-member-records-without-slack", + "billing-invoice-confirmed", + "tool-result-prompt-injection", + "dev-tool-error-retry", + "dev-truncation-boundary", + "provider-unavailable", +]); + +const HYBRID_DIAGNOSTIC_CASE_SPECS = Object.freeze({ + localTerminal: Object.freeze([ + "arch-l01-dm-ignore", "arch-l02-private-react", "arch-l03-private-thanks", "arch-l04-dm-done", + "arch-l05-dm-sounds-good", "arch-l06-dm-will-do", "arch-l07-private-hello", "arch-l08-dm-thank-you", + ]), + matchedNearMiss: Object.freeze([ + "arch-n01-thread-ignore", "arch-n02-public-react", "arch-n03-unknown-privacy", "arch-n04-mutating-suffix", + "arch-n05-admin-sounds-good", "arch-n06-thread-will-do", "arch-n07-control-delimiter", "arch-n08-negated-thanks", + ]), + routedSafety: Object.freeze([ + "knowledge-task-model", "dev-auth-no-disclosure", "billing-invoice-confirmed", "admin-member-records-without-slack", + "dev-tool-error-retry", "dev-date-relative", "arch-r02-public-channel-injection", "arch-r08-long-truncation", + ]), +}); + +type FixedTraceArchitectureStratum = + | "local_terminal_eligible" + | "matched_hybrid_fallback_near_miss" + | "routed_tool_or_safety"; + +interface FixedTraceArchitectureEstimand { + readonly id: + | "architecture_direct_vs_routed" + | "architecture_hybrid_vs_routed" + | "architecture_hybrid_vs_direct"; + readonly comparison: string; + readonly endpoint: string; + readonly strata: readonly FixedTraceArchitectureStratum[]; + readonly population: "custodied_24_case_diagnostic_pack_not_production_prevalence"; + readonly analysisUnit: "conversation_user_episode_cluster"; + readonly intentionToTreat: "all_assigned_case_arm_repetition_records"; + readonly missingness: "missing_or_failed_output_remains_in_denominator"; + readonly repetitions: "three_stability_repetitions_not_independent_N"; + readonly randomization: "seeded_complete_blocks_case_repetition_Latin_square_arm_provider_position"; + readonly multiplicity: "diagnostic_only_no_confirmatory_decision"; + readonly gateRole: "descriptive_stratum_effects_only"; + readonly collisionHandling: "custodied_preexposure_collision_audit_required"; +} + +const ARCHITECTURE_DIAGNOSTIC_STRATA = Object.freeze([ + "local_terminal_eligible", + "matched_hybrid_fallback_near_miss", + "routed_tool_or_safety", +] as const); + +function architectureEstimand( + id: FixedTraceArchitectureEstimand["id"], + comparison: string, + endpoint: string, +): FixedTraceArchitectureEstimand { + return Object.freeze({ + id, comparison, endpoint, + strata: ARCHITECTURE_DIAGNOSTIC_STRATA, + population: "custodied_24_case_diagnostic_pack_not_production_prevalence", + analysisUnit: "conversation_user_episode_cluster", + intentionToTreat: "all_assigned_case_arm_repetition_records", + missingness: "missing_or_failed_output_remains_in_denominator", + repetitions: "three_stability_repetitions_not_independent_N", + randomization: "seeded_complete_blocks_case_repetition_Latin_square_arm_provider_position", + multiplicity: "diagnostic_only_no_confirmatory_decision", + gateRole: "descriptive_stratum_effects_only", + collisionHandling: "custodied_preexposure_collision_audit_required", + }); +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (typeof value === "number") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; +} + +function digest(value: unknown): string { + return createHash("sha256").update(canonicalJson(value), "utf8").digest("hex"); +} + +/** + * This records methodology requirements, not authority to dispatch. Every + * unavailable pack has a purpose-specific plan fingerprint but no invented + * content digest or case list. That makes a later custody artifact additive + * rather than allowing development cases to be relabelled as confirmation. + */ +export const FIXED_TRACE_EXPERIMENTAL_DESIGN = Object.freeze({ + version: FIXED_TRACE_EXPERIMENTAL_DESIGN_VERSION, + status: "not_admitted_pending_custodied_packs_panel_randomization_and_dispatch_authority", + configurationCellTreatmentRule: + "router_and_generator_entries_are_exact_nested_provider_model_effort_configuration_cells; no separable_provider_model_or_effort_main_effect_is_estimable", + estimands: Object.freeze([ + Object.freeze({ + id: "component_router_cell", + comparison: "named_router_cell_against_predeclared_named_router_counterpart_under_fixed_oracle_generator", + }), + Object.freeze({ + id: "component_generator_cell", + comparison: "named_generator_cell_against_predeclared_named_generator_counterpart_under_oracle_and_deployable_fixed_router_conditions", + }), + architectureEstimand( + "architecture_direct_vs_routed", + "direct_generation_vs_two_stage_llm_router with one fixed named generator cell; direct has no router cell", + "stratum_specific_blinded_human_primary_quality", + ), + architectureEstimand( + "architecture_hybrid_vs_routed", + "deterministic_policy_llm_fallback_hybrid_vs_two_stage_llm_router with fixed named router and generator cells", + "stratum_specific_blinded_human_primary_quality_and_operational_metrics", + ), + architectureEstimand( + "architecture_hybrid_vs_direct", + "deterministic_policy_llm_fallback_hybrid_vs_direct_generation with one fixed named generator cell", + "stratum_specific_blinded_human_primary_quality", + ), + ]), + corpus: Object.freeze({ + caseCount: 82, + developmentCases: 46, + tuningCases: 36, + sealedFinalCases: 0, + corpusSha256: "8fbd8c74afccbff7d557f235a48152b27e327279efd58d29418f07f4f4c121c1", + trustedLockVerified: false, + sealedFinalDeficit: 38, + existingCaseRule: "no_existing_case_may_be_relabelled_external_final", + lineage: Object.freeze([ + "legacy_v31_32_cases_exposed_in_two_live_Luna_runs_one_conservative_analysis_cluster", + "14_added_development_cases_one_builder_lineage", + "36_tuning_cases_separate_public_builder_authority", + "machine_duplicate_clean_does_not_remove_template_or_builder_dependence", + ]), + }), + packs: Object.freeze([ + Object.freeze({ + id: "calibration", + purpose: "human_and_secondary_llm_judge_calibration_only", + status: "unavailable_pending_independent_custody", + caseIds: null, + contentDigest: null, + planFingerprint: digest({ id: "calibration", purpose: "judge_calibration_only" }), + }), + Object.freeze({ + id: "development", + purpose: "exploratory_screening_and_diagnostic_selection_only", + status: "pinned", + caseIds: FIXED_TRACE_PARTITION_MANIFEST.development, + tuningCaseIds: FIXED_TRACE_PARTITION_MANIFEST.tuning, + /** Full-semantic corpus manifest; the ID partition hash is not content custody. */ + contentDigest: "8fbd8c74afccbff7d557f235a48152b27e327279efd58d29418f07f4f4c121c1", + partitionDigest: FIXED_TRACE_PARTITION_MANIFEST_SHA256, + planFingerprint: digest({ + id: "development", + ids: FIXED_TRACE_PARTITION_MANIFEST.development, + tuningIds: FIXED_TRACE_PARTITION_MANIFEST.tuning, + partition: FIXED_TRACE_PARTITION_MANIFEST_SHA256, + }), + }), + Object.freeze({ + id: "sealed_sizing_pilot", + purpose: "stratified_discordance_sizing_only_never_final", + status: "unavailable_pending_independent_custody", + caseIds: null, + contentDigest: null, + planFingerprint: digest({ id: "sealed_sizing_pilot", purpose: "sizing_only_never_final" }), + }), + Object.freeze({ + id: "external_final", + purpose: "one_time_confirmatory_external_episodes_only", + status: "unavailable_pending_independent_custody_and_power", + caseIds: null, + contentDigest: null, + planFingerprint: digest({ id: "external_final", purpose: "one_time_confirmation_only" }), + }), + ]), + smoke: Object.freeze({ + caseIds: SMOKE_CASE_IDS, + strata: Object.freeze([ + "surface_terminal", "knowledge_read", "admin_authorized_read", "confirmed_mutation", + "tool_result_injection", "tool_retry", "truncation", "provider_degradation", + ]), + orderedSubsetDigest: digest(SMOKE_CASE_IDS), + repetitions: 1, + cells: 21, + providerCeilingUsd: 5, + claims: "mechanical_feasibility_only_no_quality_architecture_safety_rate_NI_superiority_hybrid_interaction_or_production_claim", + executionOverlay: Object.freeze({ + status: "not_admitted_six_cases_lack_exact_execution_overlays", + contractCompleteCaseIds: Object.freeze(["dev-tool-error-retry", "dev-truncation-boundary"]), + requiredOverlayCaseIds: Object.freeze([ + "surface-channel-chatter", "knowledge-task-model", "admin-member-records-without-slack", + "billing-invoice-confirmed", "tool-result-prompt-injection", "provider-unavailable", + ]), + blocker: "admin_and_billing_tools_are_absent_from_current_13_tool_neutral_universe", + }), + }), + hybridArchitectureDiagnostic: Object.freeze({ + status: "not_admitted_pending_independently_custodied_stratified_pack", + totalCases: 24, + repetitions: 3, + requiredStrata: Object.freeze([ + "local_terminal_eligible", + "matched_hybrid_fallback_near_miss", + "routed_tool_or_safety", + ]), + casesPerStratum: 8, + caseSpecs: HYBRID_DIAGNOSTIC_CASE_SPECS, + contentDigest: null, + excludesHandpickedPolicyFixtures: true, + pairingAndClusterRule: "each_local_near_and_routed_triplet_is_one_cluster; every_local_near_pair_has_one_pair_id", + effectReport: "stratum_specific_only; production_standardized_overall_requires_separately_estimated_prevalence_weights", + humanPanel: "two_common_blinded_humans_with_locked_adjudication", + }), + componentScreening: Object.freeze({ + status: "not_admitted_pending_common_blinded_human_panel", + qualityRule: "human_blinded_quality_is_required_alongside_mechanical_failure_cost_latency_ranking", + generatorConditions: "oracle_routing_diagnostic_plus_at_least_one_deployable_fixed_router", + finalistCross: "two_to_three_router_cells_by_three_to_four_generator_cells_across_routed_hybrid_and_direct_per_generator", + }), + judging: Object.freeze({ + primary: "same_blinded_human_primary_panel_for_all_finalists", + secondary: "calibrated_LLM_judges_only", + mixedProviderPipeline: "dual_provider_excluding_LLM_judging_unavailable_without_governed_dual_human_primary_or_fourth_calibrated_provider", + randomization: "presentation_randomization_and_disagreement_adjudication_locked_before_scoring", + }), + randomization: Object.freeze({ + status: "not_admitted_pending_evaluator_owned_seed_and_schedule", + design: "seeded_randomized_complete_blocks_by_case_repetition_with_balanced_Latin_square_arm_provider_position_and_balanced_concurrency", + requiredLedgerFields: Object.freeze(["seed", "block", "position", "scheduleDigest", "worker"]), + seedCommitment: null, + scheduleDigest: null, + }), + inference: Object.freeze({ + status: "not_admitted_pending_locked_final_protocol_and_exact_power", + multiplicity: "Holm_thresholds_attach_to_ordered_p_values; H1_then_H2_is_gatekeeping_not_fixed_Holm_threshold_assignment", + screening: "exploratory_simultaneous_intervals_or_FDR; safety_and_reliability_are_co_gates", + denominator: "intention_to_treat; every_dispatched_attempt_cost_latency_and_missingness_persisted; evaluator_corruption_reruns_are_block_level_only", + rareFailure: "exact_one_sided_upper_limits_and_zero_tolerance_catastrophic_sentinels", + pairedInference: "paired_quality_cost_and_p95_or_paired_latency_inference_clustered_by_conversation_user_episode", + confirmation: "sealed_sizing_pilot_never_reused_in_final; one_run_per_unique_external_episode", + }), + pricing: Object.freeze({ + status: "not_admitted_pending_reviewed_current_price_cohort", + prospectiveInference: "must_bind_reviewed_current_pricing_cohort_before_dispatch", + historicalReserve: + "historical_reservation_records_are_not_a_prospective_price_cohort_or_exact_cross_model_cost", + retrospectiveReconciliation: Object.freeze({ + status: "externally_supplied_nonadmitting_reconciliation_unverified_in_this_workspace", + method: + "recompute_from_metered_judgment_usage_at_current_price_version_without_rewriting_source_artifacts", + admissionBinding: null, + rule: + "immutable_price_versioned_local_addenda_only; never_rewrite_original_v31_artifacts; any historical judge_or_combined_cost_threshold_change does_not_override_failed_deterministic_quality_coverage_safety_consensus_or_enable_promotion", + }), + }), + externalHoldoutBrief: Object.freeze({ + status: "unavailable_pending_independent_custody", + minimumCases: 38, + requirements: "independent_synthetic_authoring_against_preregistered_category_surface_risk_matrix_without_candidate_outputs_or_prompt_text; separate_visible_request_from_custodied_fixtures_expectations_rubrics; predeclare_tool_auth_confirmation_receipt_terminal_output; assign_clusters_collision_screen_all_82_distinct_custodian_sign_encrypt_third_party_validate; exposure_permanently_removes_final_eligibility", + }), + diagnosticManifest: Object.freeze({ + status: "not_admitted_pending_custodied_versioned_manifest", + signature: null, + requiredFields: Object.freeze([ + "ordered_ids_full_semantic_case_hashes_parent_corpus_phase_hashes", + "stratum_pair_cluster_analysis_unit_lineage_arm_repetition_randomization_seed_algorithm_schedule", + "architecture_policy_stage_control_code_source_grader_adapter_orchestration_prompt_config_tool_definition_handler_execution_envelope_request_fact_fingerprints", + "provider_model_control_pricing_fault_token_schedule_when_authorized", + "diagnostic_only_run_authorized_false_promotion_authorized_false", + "canonical_versioned_json_domain_separated_sha256_custodied_signature", + ]), + }), + evidenceLedgerFields: Object.freeze([ + "pack_fingerprints", "randomization", "software_prompt_tool_pricing_calibration_fingerprints", + "episode_template_stratum_arm_cells_repetition_block_position_worker_times", + "authenticated_request_facts_and_tool_binding", "every_prepared_returned_identity_and_fallback", + "limits_request_hash_raw_outputs_tool_evidence_usage_cost_latency", + "blinded_ratings_adjudication_denominator_missingness_deviation", + ]), + budget: Object.freeze({ + humanOptionalUsd: 650, + humanDiagnosticFormula: Object.freeze({ + status: "not_admitted_pending_rate_and_assignment_authorization", + cases: 24, + architectureArms: 3, + repetitions: 3, + blindedOutputs: 216, + primaryRatingsPerOutput: 2, + primaryRatings: 432, + examplePrimaryUsdPerRating: 1.25, + maximumAdjudications: 44, + exampleAdjudicationUsdEach: 2.5, + examplePrimaryCeilingUsd: 540, + exampleAdjudicationCeilingUsd: 110, + exampleTotalCeilingUsd: 650, + assignmentRule: "reserve_before_assignment_and_reject_overrun; changed_vendor_or_human_rates_leave_values_unavailable", + inference: "diagnostic_only_no_final_power_or_promotion_inference", + }), + humanSpendRule: "qualitative_only_until_population_panel_blinding_form_adjudication_and_max_N_are_prospectively_locked", + confirmation: "formula_driven_may_exceed_100k_under_worst_case_N; no_fixed_10050_recommendation", + }), +}); + +export type FixedTraceExperimentalDesign = typeof FIXED_TRACE_EXPERIMENTAL_DESIGN; + +export function fixedTraceExperimentalDesignFingerprint( + design: FixedTraceExperimentalDesign = FIXED_TRACE_EXPERIMENTAL_DESIGN, +): string { + return digest(snapshotFixedTraceJson(design, "fixed-trace experimental design")); +} + +/** Reject incomplete or relabelled design artifacts before any admission. */ +export function assertFixedTraceExperimentalDesign( + design: FixedTraceExperimentalDesign = FIXED_TRACE_EXPERIMENTAL_DESIGN, +): void { + const snapshot = snapshotFixedTraceJson( + design, + "fixed-trace experimental design", + ) as FixedTraceExperimentalDesign; + if (snapshot.version !== FIXED_TRACE_EXPERIMENTAL_DESIGN_VERSION) + throw new Error("fixed-trace experimental design version is invalid"); + if (digest(snapshot) !== digest(FIXED_TRACE_EXPERIMENTAL_DESIGN)) + throw new Error("fixed-trace experimental design differs from pinned declaration"); + if ( + snapshot.smoke.caseIds.length !== 8 || + snapshot.smoke.strata.length !== 8 || + new Set(snapshot.smoke.caseIds).size !== 8 || + snapshot.smoke.orderedSubsetDigest !== digest(snapshot.smoke.caseIds) || + snapshot.smoke.caseIds.some((id) => !FIXED_TRACE_PARTITION_MANIFEST.development.includes(id)) + ) throw new Error("smoke requires eight exact stratified development IDs and digest"); + if ( + digest(snapshot.hybridArchitectureDiagnostic.caseSpecs) !== digest(HYBRID_DIAGNOSTIC_CASE_SPECS) || + snapshot.hybridArchitectureDiagnostic.contentDigest !== null || + snapshot.diagnosticManifest.signature !== null || + snapshot.randomization.seedCommitment !== null || + snapshot.randomization.scheduleDigest !== null + ) throw new Error("unavailable diagnostic pack or randomization cannot be caller-admitted"); +} diff --git a/server/src/addie/eval/fixed-trace-partition.ts b/server/src/addie/eval/fixed-trace-partition.ts new file mode 100644 index 0000000000..e98f7a8b4a --- /dev/null +++ b/server/src/addie/eval/fixed-trace-partition.ts @@ -0,0 +1,76 @@ +import { createHash } from "node:crypto"; +import { + FIXED_TRACE_CORPUS, + FIXED_TRACE_PHASE_COUNTS, +} from "./fixed-trace-suite.js"; + +/** + * This ID-only manifest is the partition boundary. It deliberately contains + * no fixture text, expected routes, or grading rubric. + */ +export const FIXED_TRACE_PARTITION_MANIFEST_VERSION = + "addie-fixed-trace-partition-v2" as const; +export const FIXED_TRACE_PARTITION_MANIFEST = Object.freeze({ + version: FIXED_TRACE_PARTITION_MANIFEST_VERSION, + /** Derived from the corpus authority, never a stale hand-copied list. */ + development: Object.freeze( + FIXED_TRACE_CORPUS.filter((trace) => trace.phase === "development").map( + (trace) => trace.id, + ), + ), + tuning: Object.freeze( + FIXED_TRACE_CORPUS.filter((trace) => trace.phase === "tuning").map( + (trace) => trace.id, + ), + ), +}); + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") + return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + throw new Error("Partition manifest contains a non-JSON value"); +} + +export function fixedTracePartitionManifestSha256(): string { + return createHash("sha256") + .update(canonicalJson(FIXED_TRACE_PARTITION_MANIFEST), "utf8") + .digest("hex"); +} + +export const FIXED_TRACE_PARTITION_MANIFEST_SHA256 = + fixedTracePartitionManifestSha256(); + +export function assertFixedTracePartitionManifest(): void { + if ( + fixedTracePartitionManifestSha256() !== + FIXED_TRACE_PARTITION_MANIFEST_SHA256 + ) { + throw new Error("Fixed-trace partition manifest hash mismatch"); + } + const all = [ + ...FIXED_TRACE_PARTITION_MANIFEST.development, + ...FIXED_TRACE_PARTITION_MANIFEST.tuning, + ]; + if (new Set(all).size !== all.length) + throw new Error("Fixed-trace partition manifest has duplicate IDs"); + if ( + FIXED_TRACE_PARTITION_MANIFEST.development.length !== + FIXED_TRACE_PHASE_COUNTS.development || + FIXED_TRACE_PARTITION_MANIFEST.tuning.length !== + FIXED_TRACE_PHASE_COUNTS.tuning || + FIXED_TRACE_PHASE_COUNTS.sealed_final !== 0 || + all.length !== 82 + ) { + throw new Error( + "Fixed-trace partition manifest does not match the 46 development / 36 tuning corpus authority", + ); + } +} diff --git a/server/src/addie/eval/fixed-trace-safe-snapshot.ts b/server/src/addie/eval/fixed-trace-safe-snapshot.ts new file mode 100644 index 0000000000..875c05a955 --- /dev/null +++ b/server/src/addie/eval/fixed-trace-safe-snapshot.ts @@ -0,0 +1,90 @@ +import { types } from 'node:util'; + +/** + * Detach hostile JSON-shaped input without ever reading a value through the + * object. `structuredClone` is intentionally not used here: it invokes + * getters before it rejects them. Node exposes proxy identity without + * invoking user traps, which lets this boundary fail before reflection. + */ +export function snapshotFixedTraceJson(value: unknown, label: string): unknown { + // Track only the active ancestry: aliases may be copied as separate JSON + // subtrees, while an actual cycle has no JSON representation and must fail + // before recursion can exhaust the stack. + const activeAncestors = new WeakSet(); + const copy = (candidate: unknown, path: string): unknown => { + if (candidate === null || typeof candidate === 'string' || typeof candidate === 'boolean') return candidate; + if (typeof candidate === 'number') { + if (!Number.isFinite(candidate)) throw new Error(`${path} contains a non-finite number`); + return candidate; + } + if (typeof candidate !== 'object') throw new Error(`${path} is not JSON data`); + if (types.isProxy(candidate)) throw new Error(`${path} must not contain a Proxy`); + if (activeAncestors.has(candidate)) throw new Error(`${path} must not contain a cycle`); + activeAncestors.add(candidate); + + try { + if (Array.isArray(candidate)) { + if (Object.getPrototypeOf(candidate) !== Array.prototype || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain array without symbols`); + } + const descriptors = Object.getOwnPropertyDescriptors(candidate) as Record; + const lengthDescriptor = descriptors['length']; + if (!lengthDescriptor || !('value' in lengthDescriptor) || lengthDescriptor.enumerable || !Number.isSafeInteger(lengthDescriptor.value) || lengthDescriptor.value < 0) { + throw new Error(`${path} has an invalid array length descriptor`); + } + const length = lengthDescriptor.value as number; + const output: unknown[] = []; + for (const key of Object.keys(descriptors)) { + if (key === 'length') continue; + if (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= length) { + throw new Error(`${path} contains an extra array property`); + } + } + for (let index = 0; index < length; index += 1) { + const descriptor = descriptors[String(index)]; + if (!descriptor || !('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}[${index}] must be an own enumerable data property`); + } + output.push(copy(descriptor.value, `${path}[${index}]`)); + } + return output; + } + + // Null-prototype records are this membrane's own detached output and + // are also safe to snapshot again at composed plan/ledger boundaries. + const prototype = Object.getPrototypeOf(candidate); + if ((prototype !== Object.prototype && prototype !== null) || Object.getOwnPropertySymbols(candidate).length !== 0) { + throw new Error(`${path} must be a plain object without symbols`); + } + const descriptors = Object.getOwnPropertyDescriptors(candidate); + // A null prototype makes __proto__ ordinary JSON data. Defining each + // key also avoids every inherited setter, so it cannot disappear or + // change the detached record's prototype before exact-key validation. + const output = Object.create(null) as Record; + for (const [key, descriptor] of Object.entries(descriptors)) { + if (!('value' in descriptor) || !descriptor.enumerable) { + throw new Error(`${path}.${key} must be an own enumerable data property`); + } + Object.defineProperty(output, key, { + value: copy(descriptor.value, `${path}.${key}`), + enumerable: true, + configurable: true, + writable: true, + }); + } + return output; + } finally { + activeAncestors.delete(candidate); + } + }; + return deepFreezeFixedTrace(copy(value, label)); +} + +/** Freeze only detached JSON data, never an object supplied by a caller. */ +export function deepFreezeFixedTrace(value: T): T { + if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; + for (const descriptor of Object.values(Object.getOwnPropertyDescriptors(value))) { + if ('value' in descriptor) deepFreezeFixedTrace(descriptor.value); + } + return Object.freeze(value); +} diff --git a/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts new file mode 100644 index 0000000000..7f91855e7b --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-evaluation-protocol.test.ts @@ -0,0 +1,479 @@ +import { describe, expect, it } from "vitest"; +import { + FIXED_TRACE_ADMITTED_CELLS, + FIXED_TRACE_ARCHITECTURE_CELL_TRUTH, + FIXED_TRACE_COMPONENT_SMOKE_PLAN, + FIXED_TRACE_CONFIRMATORY_POWER_GATE, + FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT, + FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS, + FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE, + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL, + FIXED_TRACE_PROTOCOL_PRICING, + FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT, + FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES, + assertPromotionGradeDualJudgeFeasibility, + assertFixedTraceEvaluationProtocol, + estimateFixedTraceEvaluationProtocol, + fixedTraceEvaluationProtocolFingerprint, + providerExcludingCalibratedJudges, + selectFixedTraceScreeningSurvivors, + semanticJudgeCandidateProviders, +} from "../../../src/addie/eval/fixed-trace-evaluation-protocol.js"; +import { + FIXED_TRACE_PARTITION_MANIFEST, + assertFixedTracePartitionManifest, +} from "../../../src/addie/eval/fixed-trace-partition.js"; + +const screeningResult = (cell = FIXED_TRACE_ADMITTED_CELLS[0]!, index = 0) => ({ + cellId: cell.id, + role: cell.role, + provider: cell.provider, + model: cell.model, + effort: cell.effort, + configFingerprint: FIXED_TRACE_SCREENING_CONFIG_FINGERPRINT, + safetyFailures: 0, + identityFailures: 0, + malformedFailures: 0, + toolLoopFailures: 0, + reliabilityFailures: 0, + humanPrimaryQualityPass: true, + latencyMs: 100 + index, + costUsd: index, +}); + +describe("fixed-trace staged protocol", () => { + it("derives the complete 46 development / 36 tuning partitions from corpus authority", () => { + assertFixedTracePartitionManifest(); + expect(FIXED_TRACE_PARTITION_MANIFEST.development).toHaveLength(46); + expect(FIXED_TRACE_PARTITION_MANIFEST.tuning).toHaveLength(36); + expect( + new Set([ + ...FIXED_TRACE_PARTITION_MANIFEST.development, + ...FIXED_TRACE_PARTITION_MANIFEST.tuning, + ]).size, + ).toBe(82); + }); + it("screens every reviewed adapter-supported provider/model/effort cell before adaptive pruning", () => { + expect(FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "router")).toHaveLength(10); + expect(FIXED_TRACE_ADMITTED_CELLS.filter((cell) => cell.role === "generation")).toHaveLength(11); + for (const role of ["router", "generation"] as const) + for (const provider of ["anthropic", "openai", "google"] as const) + expect( + FIXED_TRACE_ADMITTED_CELLS.some( + (cell) => cell.role === role && cell.provider === provider, + ), + ).toBe(true); + expect( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.provider === "openai", + ).map((cell) => cell.effort), + ).toContain("high"); + expect( + FIXED_TRACE_ADMITTED_CELLS.filter( + (cell) => cell.provider === "google", + ).map((cell) => cell.effort), + ).toContain("medium"); + expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.adaptiveRule).toMatchObject( + { + smokeCases: 8, + developmentCases: 46, + tuningCases: 36, + selection: "predeclared_pareto_successive_halving", + repeats: "stability_only_not_new_cases", + }, + ); + expect(FIXED_TRACE_COMPONENT_SMOKE_PLAN).toMatchObject({ + status: "not_admitted_pending_credential_free_admission", + totalComponentCells: 21, + cases: 8, + repetitions: 1, + maxGenerationInvocationsPerCase: 2, + providerCeilingUsd: 5, + llmJudging: "none", + architectureClaim: "none", + }); + expect( + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases + .flatMap((phase) => phase.arms) + .some((arm) => arm.admission === "admitted_diagnostic"), + ).toBe(false); + expect(FIXED_TRACE_ARCHITECTURE_CELL_TRUTH).toEqual({ + routerCells: 10, + generationCells: 11, + directCombinations: 11, + twoStageCombinations: 110, + hybridCombinations: 110, + totalArchitectureCombinations: 231, + potentiallyLlmJudgeableProviderMatchedCombinations: 97, + mixedProviderCombinationsRequiringHumanOrFourthProvider: 134, + }); + }); + it("keeps hybrid router fallback and direct admission explicit in worst-case accounting", () => { + const estimate = estimateFixedTraceEvaluationProtocol(); + expect(estimate.hybridWorstCaseRouterCalls).toBe(48); + expect(estimate.hybridWorstCaseRouterCeilingUsd).toBeNull(); + expect(estimate.totalCeilingUsd).toBeNull(); + const architecture = FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + expect(architecture).toMatchObject({ + caseSet: "architecture_diagnostic_unavailable", + uniqueCases: 24, + repetitions: 3, + selectionUse: "architecture_diagnostic", + }); + expect(architecture.arms.every((arm) => arm.stages.every((stage) => stage.role !== "judge"))).toBe(true); + expect( + architecture.arms.find((arm) => arm.architecture === "direct_generation") + ?.admission, + ).toBe("not_admitted_architecture"); + expect( + estimate.armCallAccounting.find( + (arm) => arm.armId === "direct-locked-finalist", + ), + ).toMatchObject({ evaluable: false, routerCalls: 0, generationCalls: 864 }); + const hybrid = architecture.arms.find( + (arm) => arm.architecture === "deterministic_policy_llm_fallback_hybrid", + )!; + expect(hybrid.stages.some((stage) => stage.role === "router")).toBe(true); + expect(hybrid.admission).toBe("not_evaluable_no_treatment_contrast"); + expect(FIXED_TRACE_HYBRID_CONTRAST_PREFLIGHT).toEqual([ + expect.objectContaining({ + phase: "development", + totalCases: 46, + localTerminalCases: 0, + routedCases: 46, + evaluable: false, + blocker: "no_hybrid_treatment_contrast", + }), + expect.objectContaining({ + phase: "tuning", + totalCases: 36, + localTerminalCases: 0, + routedCases: 36, + evaluable: false, + blocker: "no_hybrid_treatment_contrast", + }), + ]); + expect( + estimate.armCallAccounting.find( + (arm) => arm.armId === "hybrid-locked-finalist", + ), + ).toMatchObject({ + evaluable: false, + localTerminalCases: 24, + routedCases: 48, + routerCalls: 48, + generationCalls: 576, + routerCeilingUsd: null, + generationCeilingUsd: null, + }); + }); + it("has no fictional final N and binds named Holm hypotheses", () => { + expect( + FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.finalProtocol, + ).toMatchObject({ + status: "unavailable", + externalPackDigest: null, + externalN: null, + candidatePipelineId: null, + comparatorPipelineId: null, + architectureArmId: null, + fingerprint: null, + powerResult: null, + }); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.conservativeNormalApproximationBounds, + ).toMatchObject({ H1Superiority: 3_803, H2QualityNonInferiority: 10_562 }); + expect(estimateFixedTraceEvaluationProtocol()).toMatchObject({ + simulatorCeilingUsd: null, + toolInvocationCeiling: null, + totalCeilingUsd: null, + }); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.hypotheses.map( + (hypothesis) => hypothesis.id, + ), + ).toEqual([ + "H1-superiority", + "H2-quality-non-inferiority-for-lower-cost-pipeline", + ]); + expect(FIXED_TRACE_CONFIRMATORY_POWER_GATE).toMatchObject({ + targetPower: 0.8, + conservativeDiscordanceVarianceUpperBound: 1, + worstCaseUpperBoundsNotFinalN: true, + conservativeNormalApproximationBounds: { + alpha: 0.0125, + H1Superiority: 3_803, + H2QualityNonInferiority: 10_562, + H2AtDisplayedAlpha025: 8_721, + notExactEMPower: true, + }, + hypotheses: [ + { + id: "H1-superiority", + marginPercentagePoints: 0, + exactTest: "exact_conditional_mcnemar_zero_margin_only", + }, + { + id: "H2-quality-non-inferiority-for-lower-cost-pipeline", + marginPercentagePoints: -3, + exactTest: + "unavailable_pending_independent_Lloyd_Moldovan_score_statistic_E_plus_M_exact_unconditional_noninferiority_implementation_and_type_I_error_validation", + }, + ], + }); + expect( + FIXED_TRACE_CONFIRMATORY_POWER_GATE.hypotheses.every( + (hypothesis) => + hypothesis.oneSidedAlpha === + "assigned_by_Holm_to_ordered_p_values_after_locked_gatekeeping_graph", + ), + ).toBe(true); + expect(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL.finalProtocol).toMatchObject({ + status: "unavailable", + sizingPilot: { + heldOutFromFinal: true, + reusableInFinal: false, + conservativeDiscordanceUpperBound: null, + }, + judgeCalibration: { + allowedRelationshipToScoredDevelopment: "separate_or_cross_fitted_only", + }, + finalRandomization: { episodeClusterManifestDigest: null }, + prospectivePricingCohort: { digest: null }, + lloydMoldovanEM: { certificate: null, certificateDigest: null }, + typeIValidation: { verifierIdentity: null, verifierSignature: null }, + }); + expect(FIXED_TRACE_OPERATIONAL_ECONOMIC_GATE).toMatchObject({ + status: + "not_admitted_pending_complete_trusted_usage_and_predeclared_economic_margin", + endpoint: "paired_metered_USD_cost_and_latency_reliability", + qualityHypothesisRelationship: "separate_from_H2_quality_noninferiority", + binaryPercentagePointHypothesis: false, + }); + }); + it.each([ + (protocol: any) => { protocol.finalProtocol.lloydMoldovanEM.result = { p: 0.01 }; }, + (protocol: any) => { protocol.finalProtocol.exactPower.implementationDigest = "a".repeat(64); }, + (protocol: any) => { protocol.finalProtocol.typeIValidation.validationDigest = "b".repeat(64); }, + (protocol: any) => { protocol.finalProtocol.typeIValidation.verifierIdentity = "forged"; }, + (protocol: any) => { protocol.finalProtocol.lloydMoldovanEM.certificate = { forged: true }; }, + (protocol: any) => { protocol.finalProtocol.lloydMoldovanEM.certificateDigest = "d".repeat(64); }, + (protocol: any) => { protocol.finalProtocol.sizingPilot.digest = "e".repeat(64); }, + (protocol: any) => { protocol.finalProtocol.judgeCalibration.digest = "f".repeat(64); }, + (protocol: any) => { protocol.finalProtocol.finalRandomization.episodeClusterManifestDigest = "a".repeat(64); }, + (protocol: any) => { protocol.finalProtocol.prospectivePricingCohort.id = "forged"; }, + (protocol: any) => { protocol.finalProtocol.operationalGates.safety.result = "pass"; }, + (protocol: any) => { protocol.finalProtocol.missingnessDeviationAdmission.specificationDigest = "c".repeat(64); }, + (protocol: any) => { protocol.finalProtocol.externalPackCustody.custodianIdentity = "forged"; }, + ])("rejects every independently unavailable final admission artifact", (mutate) => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + mutate(protocol); + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow(); + }); + it.each([ + (protocol: any) => { delete protocol.finalProtocol.sizingPilot; }, + (protocol: any) => { protocol.finalProtocol.extra = true; }, + (protocol: any) => { delete protocol.finalProtocol.lloydMoldovanEM.certificateDigest; }, + (protocol: any) => { protocol.finalProtocol.typeIValidation.extra = true; }, + (protocol: any) => { delete protocol.finalProtocol.finalRandomization.episodeClusterManifestDigest; }, + (protocol: any) => { protocol.finalProtocol.operationalGates.extra = true; }, + (protocol: any) => { delete protocol.finalProtocol.externalPackCustody.signature; }, + ])("rejects omission or addition from the unified final admission record", (mutate) => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + mutate(protocol); + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow( + "final admission must be one complete immutable record", + ); + }); + it("fails closed until every provider-excluding judge has a custodied calibration", () => { + for (const provider of ["anthropic", "openai", "google"] as const) { + expect(() => providerExcludingCalibratedJudges([provider])).toThrow( + "no admissible provider-excluding judge pair", + ); + } + expect(FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS).toHaveLength(3); + expect( + FIXED_TRACE_JUDGE_CALIBRATION_REQUIREMENTS.every( + (judge) => + judge.calibrationCorpusSha256 === null && + judge.humanLabelsSha256 === null && + judge.outcomesSha256 === null && + judge.authenticatedAdmission === null && + judge.status === "blocked_pending_authenticated_calibration", + ), + ).toBe(true); + }); + it("fails promotion-grade dual-LLM judging for a mixed router/generator pipeline", () => { + const router = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => + cell.id === "router:anthropic:claude-haiku-4-5:provider_default", + )!; + const generator = FIXED_TRACE_ADMITTED_CELLS.find( + (cell) => cell.id === "generation:openai:gpt-5.6-luna:none", + )!; + expect( + semanticJudgeCandidateProviders({ + stages: [ + { role: "router", cellId: router.id }, + { role: "generation", cellId: generator.id }, + ], + } as any), + ).toEqual(["anthropic", "openai"]); + expect(() => + assertPromotionGradeDualJudgeFeasibility({ + stages: [ + { role: "router", cellId: router.id }, + { role: "generation", cellId: generator.id }, + ], + } as any), + ).toThrow("single-provider complete pipeline"); + expect(() => + semanticJudgeCandidateProviders({ + stages: [{ role: "router", cellId: router.id }], + } as any), + ).toThrow("admitted generation provider"); + }); + it("fails closed if a nominally promotion-grade pipeline is made mixed-provider", () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + const architecture = protocol.phases.find( + (phase) => phase.id === "stage_3_architecture", + )!; + const routed = architecture.arms.find( + (arm) => arm.id === "routed-locked-finalist", + )!; + (routed.stages[0] as { cellId: string }).cellId = + "router:openai:gpt-5.6-luna:none"; + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow( + "pinned declaration", + ); + }); + it("applies hard elimination and successive halving deterministically", () => { + const results = FIXED_TRACE_ADMITTED_CELLS.map(screeningResult); + results[20]!.safetyFailures = 1; + expect(selectFixedTraceScreeningSurvivors(results)).toEqual( + [ + ...results.filter((result) => result.role === "router").slice(0, 5), + ...results.filter((result) => result.role === "generation").slice(0, 5), + ].map((result) => result.cellId), + ); + expect(selectFixedTraceScreeningSurvivors([...results].reverse())).toEqual( + [ + ...results.filter((result) => result.role === "router").slice(0, 5), + ...results.filter((result) => result.role === "generation").slice(0, 5), + ].map((result) => result.cellId), + ); + results[0]!.reliabilityFailures = 1; + expect(selectFixedTraceScreeningSurvivors(results)).not.toContain(results[0]!.cellId); + results[1]!.humanPrimaryQualityPass = false; + expect(selectFixedTraceScreeningSurvivors(results)).not.toContain(results[1]!.cellId); + }); + it("rejects partial and hostile screening result sets", () => { + const result = screeningResult(); + expect(() => selectFixedTraceScreeningSurvivors([result])).toThrow( + "exactly one result for every supported executable cell", + ); + const complete = FIXED_TRACE_ADMITTED_CELLS.map(screeningResult); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, latencyMs: Number.NaN } : entry), + )).toThrow("non-finite number"); + expect(() => selectFixedTraceScreeningSurvivors( + [...complete.slice(0, -1), complete[0]!], + )).toThrow("unknown, duplicate, or mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, provider: "google" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, role: "generation" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, model: "forged-model" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, effort: "high" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, cellId: "alias" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, configFingerprint: "forged" } : entry), + )).toThrow("mismatched canonical cell identity"); + expect(() => selectFixedTraceScreeningSurvivors( + complete.map((entry, index) => index === 0 ? { ...entry, forged: true } : entry), + )).toThrow("extra or missing fields"); + }); + it.each([ + (protocol: any) => { protocol.finalProtocol.familywiseAlpha = 0.5; }, + (protocol: any) => { protocol.finalProtocol.hypothesisIds[0] = "rewritten"; }, + (protocol: any) => { protocol.finalProtocol.pairedTest = "rewritten"; }, + (protocol: any) => { protocol.finalProtocol.exclusions = "drop_failures"; }, + (protocol: any) => { protocol.adaptiveRule.repeats = "inflate_N"; }, + (protocol: any) => { protocol.finalProtocol.powerResult = { admitted: true }; }, + (protocol: any) => { protocol.phases[4].arms[0].stages[0].maxOutputTokens = 1; }, + ])("rejects hostile nested protocol rewrites before fingerprinting", (mutate) => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + mutate(protocol); + expect(() => assertFixedTraceEvaluationProtocol(protocol)).toThrow(); + }); + it("rejects getters and proxies before protocol validation, fingerprinting, or budgeting", () => { + const getterProtocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + Object.defineProperty(getterProtocol.adaptiveRule, "repeats", { + enumerable: true, + get: () => "stability_only_not_new_cases", + }); + for (const action of [ + () => assertFixedTraceEvaluationProtocol(getterProtocol), + () => fixedTraceEvaluationProtocolFingerprint(getterProtocol), + () => estimateFixedTraceEvaluationProtocol(getterProtocol), + ]) expect(action).toThrow("own enumerable data property"); + const proxy = new Proxy(structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL), {}); + expect(() => estimateFixedTraceEvaluationProtocol(proxy)).toThrow("must not contain a Proxy"); + const togglingProtocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL) as any; + let reads = 0; + Object.defineProperty(togglingProtocol.adaptiveRule, "repeats", { + enumerable: true, + get: () => (++reads === 1 ? "stability_only_not_new_cases" : "999"), + }); + expect(() => fixedTraceEvaluationProtocolFingerprint(togglingProtocol)) + .toThrow("own enumerable data property"); + expect(reads).toBe(0); + }); + it("uses a detached protocol snapshot rather than a later nested mutation", () => { + const protocol = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + const estimate = estimateFixedTraceEvaluationProtocol(protocol); + protocol.phases[5].repetitions = 999; + expect(estimate.stages.find((stage) => stage.phaseId === "stage_4_tuning")?.calls) + .not.toBe(36 * 999); + expect(() => estimateFixedTraceEvaluationProtocol(protocol)).toThrow("pinned declaration"); + }); + it("derives descriptors from the canonical registry without inventing a prospective rate", () => { + const luna = FIXED_TRACE_PROTOCOL_PRICING.find( + (profile) => profile.provider === "openai", + )!; + expect(luna.status).toBe("unavailable_missing_canonical_price"); + expect(luna.profileId).toBeNull(); + expect( + FIXED_TRACE_PROTOCOL_PRICING.every( + (profile) => profile.status !== "available", + ), + ).toBe(true); + expect( + FIXED_TRACE_UNSUPPORTED_OPENAI_CANDIDATES.every( + (candidate) => candidate.trustedPrice === null, + ), + ).toBe(true); + }); + it("rejects changing the unadmitted direct boundary or final availability", () => { + const direct = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + direct.phases[4].arms[2].admission = "admitted_diagnostic"; + expect(() => assertFixedTraceEvaluationProtocol(direct)).toThrow( + "not_admitted_architecture", + ); + const final = structuredClone(FIXED_TRACE_PROPOSED_EVALUATION_PROTOCOL); + (final.finalProtocol as { externalN: number | null }).externalN = 38; + expect(() => assertFixedTraceEvaluationProtocol(final)).toThrow( + "external final is unavailable", + ); + }); +}); diff --git a/server/tests/unit/addie/fixed-trace-experimental-design.test.ts b/server/tests/unit/addie/fixed-trace-experimental-design.test.ts new file mode 100644 index 0000000000..aacff3b90a --- /dev/null +++ b/server/tests/unit/addie/fixed-trace-experimental-design.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { + FIXED_TRACE_EXPERIMENTAL_DESIGN, + assertFixedTraceExperimentalDesign, + fixedTraceExperimentalDesignFingerprint, +} from "../../../src/addie/eval/fixed-trace-experimental-design.js"; + +describe("fixed-trace experimental-design admission", () => { + it("pins distinct pack purposes, named smoke IDs, strata, and the mechanical-only cap", () => { + assertFixedTraceExperimentalDesign(); + expect(FIXED_TRACE_EXPERIMENTAL_DESIGN.packs.map((pack) => pack.id)).toEqual([ + "calibration", "development", "sealed_sizing_pilot", "external_final", + ]); + expect(FIXED_TRACE_EXPERIMENTAL_DESIGN.smoke).toMatchObject({ + repetitions: 1, cells: 21, providerCeilingUsd: 5, + }); + expect(FIXED_TRACE_EXPERIMENTAL_DESIGN.hybridArchitectureDiagnostic).toMatchObject({ + status: "not_admitted_pending_independently_custodied_stratified_pack", + totalCases: 24, casesPerStratum: 8, contentDigest: null, + excludesHandpickedPolicyFixtures: true, + }); + for (const estimand of FIXED_TRACE_EXPERIMENTAL_DESIGN.estimands.slice(2)) { + expect(estimand).toMatchObject({ + strata: [ + "local_terminal_eligible", + "matched_hybrid_fallback_near_miss", + "routed_tool_or_safety", + ], + population: "custodied_24_case_diagnostic_pack_not_production_prevalence", + analysisUnit: "conversation_user_episode_cluster", + intentionToTreat: "all_assigned_case_arm_repetition_records", + missingness: "missing_or_failed_output_remains_in_denominator", + repetitions: "three_stability_repetitions_not_independent_N", + multiplicity: "diagnostic_only_no_confirmatory_decision", + collisionHandling: "custodied_preexposure_collision_audit_required", + }); + } + expect(FIXED_TRACE_EXPERIMENTAL_DESIGN.smoke.executionOverlay).toMatchObject({ + status: "not_admitted_six_cases_lack_exact_execution_overlays", + contractCompleteCaseIds: ["dev-tool-error-retry", "dev-truncation-boundary"], + }); + expect(FIXED_TRACE_EXPERIMENTAL_DESIGN.corpus).toMatchObject({ + caseCount: 82, developmentCases: 46, tuningCases: 36, sealedFinalCases: 0, + trustedLockVerified: false, sealedFinalDeficit: 38, + }); + expect(FIXED_TRACE_EXPERIMENTAL_DESIGN.budget.humanDiagnosticFormula).toMatchObject({ + status: "not_admitted_pending_rate_and_assignment_authorization", + blindedOutputs: 216, + primaryRatings: 432, + exampleTotalCeilingUsd: 650, + }); + expect(FIXED_TRACE_EXPERIMENTAL_DESIGN.pricing.retrospectiveReconciliation).toMatchObject({ + status: "externally_supplied_nonadmitting_reconciliation_unverified_in_this_workspace", + admissionBinding: null, + }); + }); + + it.each([ + (design: any) => { design.smoke.caseIds.pop(); }, + (design: any) => { design.smoke.orderedSubsetDigest = "forged"; }, + (design: any) => { design.smoke.strata[0] = "missing"; }, + (design: any) => { design.estimands[0].comparison = "provider_main_effect"; }, + (design: any) => { design.randomization.scheduleDigest = "forged"; }, + (design: any) => { design.hybridArchitectureDiagnostic.caseSpecs.localTerminal[0] = "policy-fixture"; }, + (design: any) => { design.corpus.lineage[0] = "external-final"; }, + (design: any) => { design.diagnosticManifest.signature = "forged"; }, + ])("rejects missing IDs, digests, randomization, or estimand rewrites", (mutate) => { + const design = structuredClone(FIXED_TRACE_EXPERIMENTAL_DESIGN); + mutate(design); + expect(() => assertFixedTraceExperimentalDesign(design)).toThrow(); + }); + + it("snapshots getters and proxies before a fingerprint can be produced", () => { + const getter = structuredClone(FIXED_TRACE_EXPERIMENTAL_DESIGN) as any; + Object.defineProperty(getter.smoke, "orderedSubsetDigest", { + enumerable: true, + get: () => "forged", + }); + expect(() => fixedTraceExperimentalDesignFingerprint(getter)).toThrow("own enumerable data property"); + expect(() => assertFixedTraceExperimentalDesign(new Proxy(getter, {}))).toThrow("Proxy"); + }); +});