From 756c6c58c4f97760b167109bc2e49e42b5711746 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 09:26:58 +0000 Subject: [PATCH] fix(experiment): judge preflight + failed-run data preservation (run-002 attempt-1 post-mortem) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run-002 attempt 1 failed: OPENWEIGHT_API_KEY present but INVALID (Together 401 on every call). Two design gaps let that sink the run: 1. --skip-missing only checked env-var PRESENCE, so the broken judge burned 1252 failed calls, pushed the failure rate to 33% > 20%, and the CLI (correctly) refused to bless the run. NEW: preflightJudges() makes ONE live probe call per judge before its ~1250 scoring calls; any preflight failure (invalid key, wrong model id, unreachable endpoint) skips that judge and records it as a protocol deviation — the run proceeds with the judges that actually work. All-fail → loud refusal with a checklist message. 3 new tests. 2. The workflow's publish step did not run after a failed experiment step, so the ~2500 PAID Anthropic+OpenAI calls of attempt 1 were written to the runner disk and lost. NEW: publish runs if: always(); a failed run's data is pushed on an experiment/-FAILED- branch for diagnosis — with NO automatic pull request (a failed run must never be offered as results), and a no-output guard when the run died before writing anything. Diagnosis note recorded: OPENWEIGHT_BASE_URL was CORRECT (the 401 is Together's own response body — the endpoint was reached); the API key secret itself is invalid and must be replaced. Validation: web 414/414, core 213/213, cold tsc clean, sync no drift. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y7wiMwFa2D4P8zD9RhCgsv --- .github/workflows/experiment.yml | 28 +++++++++++++- apps/web/cli/__tests__/experiment-lib.test.ts | 38 ++++++++++++++++++- apps/web/cli/experiment-lib.ts | 31 +++++++++++++++ apps/web/cli/experiment.ts | 15 +++++--- 4 files changed, 105 insertions(+), 7 deletions(-) diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index 3078d2b..e3a152c 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -53,6 +53,7 @@ jobs: npm ci npx prisma generate - name: Execute experiment + id: exp working-directory: apps/web env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -63,19 +64,44 @@ jobs: MOCK_FLAG="" if [ "$MOCK_INPUT" = "true" ]; then MOCK_FLAG="--mock"; fi npx tsx cli/experiment.ts -c "${GITHUB_WORKSPACE}/${CONFIG_PATH}" $MOCK_FLAG --skip-missing + # Runs even when the experiment step fails: run-002 attempt 1 lost + # ~2500 paid judge calls because the failed step blocked publication. + # Failed-run data is diagnostic material — pushed on a clearly-labeled + # branch, with NO automatic pull request (a failed run must never look + # like results). - name: Publish results branch (PR automatic, or one-click link) + if: ${{ always() && steps.exp.outcome != 'skipped' }} env: GH_TOKEN: ${{ github.token }} + EXP_OUTCOME: ${{ steps.exp.outcome }} run: | RUN_DIR=$(dirname "$CONFIG_PATH") - SUFFIX=$([ "$MOCK_INPUT" = "true" ] && echo "mock" || echo "results") + if [ "$EXP_OUTCOME" != "success" ]; then + SUFFIX="FAILED" + elif [ "$MOCK_INPUT" = "true" ]; then + SUFFIX="mock" + else + SUFFIX="results" + fi BRANCH="experiment/$(basename "$RUN_DIR")-${SUFFIX}-${GITHUB_RUN_ID}" + if [ -z "$(git status --porcelain "$RUN_DIR")" ]; then + echo "### ⚠️ No experiment output to publish (the run failed before writing results)" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi git config user.name "caims-experiment-bot" git config user.email "experiments@users.noreply.github.com" git checkout -b "$BRANCH" git add "$RUN_DIR" git commit -m "results($(basename "$RUN_DIR")): ${SUFFIX} from workflow run ${GITHUB_RUN_ID}" git push -u origin "$BRANCH" + # A FAILED run's data is preserved for diagnosis but never offered + # as results: branch only, no pull request. + if [ "$EXP_OUTCOME" != "success" ]; then + echo "### ❌ Experiment FAILED — partial data preserved for diagnosis (never merge as results)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Branch: \`$BRANCH\` — inspect summary.json / raw JSONL to see which judge failed and why." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi # Automatic PR needs the repo toggle "Allow GitHub Actions to create # and approve pull requests". If it is off, we do NOT fail: the # results branch is already pushed — we surface a one-click link in diff --git a/apps/web/cli/__tests__/experiment-lib.test.ts b/apps/web/cli/__tests__/experiment-lib.test.ts index f7c8194..be90f5c 100644 --- a/apps/web/cli/__tests__/experiment-lib.test.ts +++ b/apps/web/cli/__tests__/experiment-lib.test.ts @@ -1,5 +1,5 @@ import { - ExperimentConfigSchema, runExperiment, createMockAdapter, + ExperimentConfigSchema, runExperiment, createMockAdapter, preflightJudges, } from '../experiment-lib'; import type { ExperimentConfig, DatasetItem, SampleRecord } from '../experiment-lib'; @@ -331,3 +331,39 @@ describe('partitionJudgesByEnv (--skip-missing support)', () => { expect(summary.skippedJudges).toEqual([{ id: 'gpt-4o', reason: 'env var OPENAI_API_KEY not set' }]); }); }); + +describe('preflightJudges — one live probe per judge before ~1250 scoring calls', () => { + const judgeA = { id: 'good', provider: 'anthropic' as const, model: 'model-a', apiKeyEnv: 'KEY_A' }; + const judgeB = { id: 'bad-key', provider: 'openai-compatible' as const, model: 'model-b', apiKeyEnv: 'KEY_B', baseUrlEnv: 'URL_B' }; + + it('a judge whose probe succeeds is runnable; a 401 judge is skipped with the reason recorded', async () => { + const adapterFor = (j: { id: string }) => ({ + chat: jest.fn(), + judge: j.id === 'bad-key' + ? jest.fn().mockRejectedValue(new Error('OpenAI API error 401: Invalid API key provided')) + : jest.fn().mockResolvedValue('OK'), + }); + const { runnable, skipped } = await preflightJudges([judgeA, judgeB], adapterFor); + expect(runnable.map(j => j.id)).toEqual(['good']); + expect(skipped).toHaveLength(1); + expect(skipped[0].id).toBe('bad-key'); + expect(skipped[0].reason).toContain('preflight call failed'); + expect(skipped[0].reason).toContain('401'); + }); + + it('all judges failing preflight → empty runnable (the caller must refuse to run)', async () => { + const adapterFor = () => ({ + chat: jest.fn(), + judge: jest.fn().mockRejectedValue(new Error('connect ECONNREFUSED')), + }); + const { runnable, skipped } = await preflightJudges([judgeA, judgeB], adapterFor); + expect(runnable).toHaveLength(0); + expect(skipped).toHaveLength(2); + }); + + it('probe uses the judge model and a tiny token cap (cost of a preflight ~ nothing)', async () => { + const judgeFn = jest.fn().mockResolvedValue('OK'); + await preflightJudges([judgeA], () => ({ chat: jest.fn(), judge: judgeFn })); + expect(judgeFn).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ model: 'model-a', maxTokens: 8 })); + }); +}); diff --git a/apps/web/cli/experiment-lib.ts b/apps/web/cli/experiment-lib.ts index 590c324..2d81881 100644 --- a/apps/web/cli/experiment-lib.ts +++ b/apps/web/cli/experiment-lib.ts @@ -162,6 +162,37 @@ export function partitionJudgesByEnv( return { runnable, skipped }; } +// One live probe call per judge BEFORE its ~1250 scoring calls. Run-002 +// attempt 1 burned 1252 failed calls because a PRESENT-but-INVALID key +// (Together 401) passed the env-var check and then failed every call, +// pushing the run over the failure threshold and losing the two good +// judges' data. A preflight failure of any kind (bad key, wrong model id, +// unreachable endpoint) means every scoring call would fail identically, +// so the judge is skipped and recorded as a protocol deviation instead. +export async function preflightJudges( + judges: JudgeConfig[], + adapterFor: (judge: JudgeConfig) => LLMAdapter, + log?: (msg: string) => void +): Promise<{ runnable: JudgeConfig[]; skipped: { id: string; reason: string }[] }> { + const runnable: JudgeConfig[] = []; + const skipped: { id: string; reason: string }[] = []; + for (const judge of judges) { + try { + await adapterFor(judge).judge( + 'Preflight credential check. Reply with the single word OK.', + { model: judge.model, maxTokens: 8 } + ); + runnable.push(judge); + log?.(`preflight: judge "${judge.id}" ok`); + } catch (error) { + const msg = (error instanceof Error ? error.message : String(error)).slice(0, 200); + skipped.push({ id: judge.id, reason: `preflight call failed: ${msg}` }); + log?.(`preflight: judge "${judge.id}" FAILED — skipped, recorded as protocol deviation (${msg})`); + } + } + return { runnable, skipped }; +} + // ── Runner ──────────────────────────────────────────────────────────────── export interface RunDeps { diff --git a/apps/web/cli/experiment.ts b/apps/web/cli/experiment.ts index b6b1911..e80f0f0 100644 --- a/apps/web/cli/experiment.ts +++ b/apps/web/cli/experiment.ts @@ -17,7 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { - ExperimentConfigSchema, runExperiment, createMockAdapter, partitionJudgesByEnv, + ExperimentConfigSchema, runExperiment, createMockAdapter, partitionJudgesByEnv, preflightJudges, } from './experiment-lib'; import type { ExperimentConfig, JudgeConfig, DatasetItem, SampleRecord, ExperimentSummary } from './experiment-lib'; import { AnthropicAdapter } from '@/lib/adapters/anthropic'; @@ -148,9 +148,11 @@ async function main() { const rawConfig = JSON.parse(fs.readFileSync(configAbs, 'utf-8')); let config = ExperimentConfigSchema.parse(rawConfig); - // --skip-missing: run with the judges whose credentials exist; record the - // rest as a protocol deviation (protocol-001 Amendment A2). Mock runs need - // no credentials, so nothing is skipped there. + // --skip-missing: run with the judges whose credentials exist AND work; the + // rest are recorded as a protocol deviation (protocol-001 Amendment A2). + // Two gates: (1) env vars present, (2) one live preflight call per judge — + // run-002 attempt 1 showed a present-but-invalid key burns ~1250 failed + // calls and sinks the whole run. Mock runs need no credentials. let skippedJudges: { id: string; reason: string }[] = []; if (skipMissing && !mock) { const { runnable, skipped } = partitionJudgesByEnv(config.judges, process.env); @@ -159,7 +161,10 @@ async function main() { process.stderr.write(`skip-missing: judge "${sk.id}" skipped (${sk.reason})\n`); } if (runnable.length === 0) fail('all judges skipped — no credentials found; set at least ANTHROPIC_API_KEY or OPENAI_API_KEY'); - config = { ...config, judges: runnable }; + const pf = await preflightJudges(runnable, realAdapterFor, (m) => process.stderr.write(m + '\n')); + skippedJudges = [...skippedJudges, ...pf.skipped]; + if (pf.runnable.length === 0) fail('all judges failed preflight — no working credentials (check the API keys, base URLs and model ids)'); + config = { ...config, judges: pf.runnable }; } const baseOut = outDir ? path.resolve(outDir) : configDir;