fix(experiment): judge preflight + failed-run data preservation

.github/workflows/experiment.yml

@@ -53,6 +53,7 @@ jobs:
           npm ci
           npx prisma generate
       - name: Execute experiment
+        id: exp
         working-directory: apps/web
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -63,19 +64,44 @@ jobs:
           MOCK_FLAG=""
           if [ "$MOCK_INPUT" = "true" ]; then MOCK_FLAG="--mock"; fi
           npx tsx cli/experiment.ts -c "${GITHUB_WORKSPACE}/${CONFIG_PATH}" $MOCK_FLAG --skip-missing
+      # Runs even when the experiment step fails: run-002 attempt 1 lost
+      # ~2500 paid judge calls because the failed step blocked publication.
+      # Failed-run data is diagnostic material — pushed on a clearly-labeled
+      # branch, with NO automatic pull request (a failed run must never look
+      # like results).
       - name: Publish results branch (PR automatic, or one-click link)
+        if: ${{ always() && steps.exp.outcome != 'skipped' }}
         env:
           GH_TOKEN: ${{ github.token }}
+          EXP_OUTCOME: ${{ steps.exp.outcome }}
         run: |
           RUN_DIR=$(dirname "$CONFIG_PATH")
-          SUFFIX=$([ "$MOCK_INPUT" = "true" ] && echo "mock" || echo "results")
+          if [ "$EXP_OUTCOME" != "success" ]; then
+            SUFFIX="FAILED"
+          elif [ "$MOCK_INPUT" = "true" ]; then
+            SUFFIX="mock"
+          else
+            SUFFIX="results"
+          fi
           BRANCH="experiment/$(basename "$RUN_DIR")-${SUFFIX}-${GITHUB_RUN_ID}"
+          if [ -z "$(git status --porcelain "$RUN_DIR")" ]; then
+            echo "### ⚠️ No experiment output to publish (the run failed before writing results)" >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
           git config user.name "caims-experiment-bot"
           git config user.email "experiments@users.noreply.github.com"
           git checkout -b "$BRANCH"
           git add "$RUN_DIR"
           git commit -m "results($(basename "$RUN_DIR")): ${SUFFIX} from workflow run ${GITHUB_RUN_ID}"
           git push -u origin "$BRANCH"
+          # A FAILED run's data is preserved for diagnosis but never offered
+          # as results: branch only, no pull request.
+          if [ "$EXP_OUTCOME" != "success" ]; then
+            echo "### ❌ Experiment FAILED — partial data preserved for diagnosis (never merge as results)" >> "$GITHUB_STEP_SUMMARY"
+            echo "" >> "$GITHUB_STEP_SUMMARY"
+            echo "Branch: \`$BRANCH\` — inspect summary.json / raw JSONL to see which judge failed and why." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
           # Automatic PR needs the repo toggle "Allow GitHub Actions to create
           # and approve pull requests". If it is off, we do NOT fail: the
           # results branch is already pushed — we surface a one-click link in

apps/web/cli/__tests__/experiment-lib.test.ts

@@ -1,5 +1,5 @@
 import {
-  ExperimentConfigSchema, runExperiment, createMockAdapter,
+  ExperimentConfigSchema, runExperiment, createMockAdapter, preflightJudges,
 } from '../experiment-lib';
 import type { ExperimentConfig, DatasetItem, SampleRecord } from '../experiment-lib';
 
@@ -331,3 +331,39 @@ describe('partitionJudgesByEnv (--skip-missing support)', () => {
     expect(summary.skippedJudges).toEqual([{ id: 'gpt-4o', reason: 'env var OPENAI_API_KEY not set' }]);
   });
 });
+
+describe('preflightJudges — one live probe per judge before ~1250 scoring calls', () => {
+  const judgeA = { id: 'good', provider: 'anthropic' as const, model: 'model-a', apiKeyEnv: 'KEY_A' };
+  const judgeB = { id: 'bad-key', provider: 'openai-compatible' as const, model: 'model-b', apiKeyEnv: 'KEY_B', baseUrlEnv: 'URL_B' };
+
+  it('a judge whose probe succeeds is runnable; a 401 judge is skipped with the reason recorded', async () => {
+    const adapterFor = (j: { id: string }) => ({
+      chat: jest.fn(),
+      judge: j.id === 'bad-key'
+        ? jest.fn().mockRejectedValue(new Error('OpenAI API error 401: Invalid API key provided'))
+        : jest.fn().mockResolvedValue('OK'),
+    });
+    const { runnable, skipped } = await preflightJudges([judgeA, judgeB], adapterFor);
+    expect(runnable.map(j => j.id)).toEqual(['good']);
+    expect(skipped).toHaveLength(1);
+    expect(skipped[0].id).toBe('bad-key');
+    expect(skipped[0].reason).toContain('preflight call failed');
+    expect(skipped[0].reason).toContain('401');
+  });
+
+  it('all judges failing preflight → empty runnable (the caller must refuse to run)', async () => {
+    const adapterFor = () => ({
+      chat: jest.fn(),
+      judge: jest.fn().mockRejectedValue(new Error('connect ECONNREFUSED')),
+    });
+    const { runnable, skipped } = await preflightJudges([judgeA, judgeB], adapterFor);
+    expect(runnable).toHaveLength(0);
+    expect(skipped).toHaveLength(2);
+  });
+
+  it('probe uses the judge model and a tiny token cap (cost of a preflight ~ nothing)', async () => {
+    const judgeFn = jest.fn().mockResolvedValue('OK');
+    await preflightJudges([judgeA], () => ({ chat: jest.fn(), judge: judgeFn }));
+    expect(judgeFn).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ model: 'model-a', maxTokens: 8 }));
+  });
+});

apps/web/cli/experiment-lib.ts

@@ -162,6 +162,37 @@ export function partitionJudgesByEnv(
   return { runnable, skipped };
 }
 
+// One live probe call per judge BEFORE its ~1250 scoring calls. Run-002
+// attempt 1 burned 1252 failed calls because a PRESENT-but-INVALID key
+// (Together 401) passed the env-var check and then failed every call,
+// pushing the run over the failure threshold and losing the two good
+// judges' data. A preflight failure of any kind (bad key, wrong model id,
+// unreachable endpoint) means every scoring call would fail identically,
+// so the judge is skipped and recorded as a protocol deviation instead.
+export async function preflightJudges(
+  judges: JudgeConfig[],
+  adapterFor: (judge: JudgeConfig) => LLMAdapter,
+  log?: (msg: string) => void
+): Promise<{ runnable: JudgeConfig[]; skipped: { id: string; reason: string }[] }> {
+  const runnable: JudgeConfig[] = [];
+  const skipped: { id: string; reason: string }[] = [];
+  for (const judge of judges) {
+    try {
+      await adapterFor(judge).judge(
+        'Preflight credential check. Reply with the single word OK.',
+        { model: judge.model, maxTokens: 8 }
+      );
+      runnable.push(judge);
+      log?.(`preflight: judge "${judge.id}" ok`);
+    } catch (error) {
+      const msg = (error instanceof Error ? error.message : String(error)).slice(0, 200);
+      skipped.push({ id: judge.id, reason: `preflight call failed: ${msg}` });
+      log?.(`preflight: judge "${judge.id}" FAILED — skipped, recorded as protocol deviation (${msg})`);
+    }
+  }
+  return { runnable, skipped };
+}
+
 // ── Runner ────────────────────────────────────────────────────────────────
 
 export interface RunDeps {

apps/web/cli/experiment.ts

@@ -17,7 +17,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import {
-  ExperimentConfigSchema, runExperiment, createMockAdapter, partitionJudgesByEnv,
+  ExperimentConfigSchema, runExperiment, createMockAdapter, partitionJudgesByEnv, preflightJudges,
 } from './experiment-lib';
 import type { ExperimentConfig, JudgeConfig, DatasetItem, SampleRecord, ExperimentSummary } from './experiment-lib';
 import { AnthropicAdapter } from '@/lib/adapters/anthropic';
@@ -148,9 +148,11 @@ async function main() {
   const rawConfig = JSON.parse(fs.readFileSync(configAbs, 'utf-8'));
   let config = ExperimentConfigSchema.parse(rawConfig);
 
-  // --skip-missing: run with the judges whose credentials exist; record the
-  // rest as a protocol deviation (protocol-001 Amendment A2). Mock runs need
-  // no credentials, so nothing is skipped there.
+  // --skip-missing: run with the judges whose credentials exist AND work; the
+  // rest are recorded as a protocol deviation (protocol-001 Amendment A2).
+  // Two gates: (1) env vars present, (2) one live preflight call per judge —
+  // run-002 attempt 1 showed a present-but-invalid key burns ~1250 failed
+  // calls and sinks the whole run. Mock runs need no credentials.
   let skippedJudges: { id: string; reason: string }[] = [];
   if (skipMissing && !mock) {
     const { runnable, skipped } = partitionJudgesByEnv(config.judges, process.env);
@@ -159,7 +161,10 @@ async function main() {
       process.stderr.write(`skip-missing: judge "${sk.id}" skipped (${sk.reason})\n`);
     }
     if (runnable.length === 0) fail('all judges skipped — no credentials found; set at least ANTHROPIC_API_KEY or OPENAI_API_KEY');
-    config = { ...config, judges: runnable };
+    const pf = await preflightJudges(runnable, realAdapterFor, (m) => process.stderr.write(m + '\n'));
+    skippedJudges = [...skippedJudges, ...pf.skipped];
+    if (pf.runnable.length === 0) fail('all judges failed preflight — no working credentials (check the API keys, base URLs and model ids)');
+    config = { ...config, judges: pf.runnable };
   }
 
   const baseOut = outDir ? path.resolve(outDir) : configDir;