feat(experiment): preflight_only credential-check mode — verify secrets with zero spend

.github/workflows/experiment.yml

@@ -23,6 +23,10 @@ on:
         description: "Run with the deterministic mock judge (pipeline validation only)"
         type: boolean
         default: false
+      preflight_only:
+        description: "Credential check only: one tiny probe call per judge, no scoring, no spend. Green = all judges ready; red = a secret needs fixing."
+        type: boolean
+        default: false
 
 permissions:
   contents: write
@@ -40,6 +44,7 @@ jobs:
       # scripts in a secret-holding workflow.
       CONFIG_PATH: ${{ inputs.config }}
       MOCK_INPUT: ${{ inputs.mock }}
+      PREFLIGHT_INPUT: ${{ inputs.preflight_only }}
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-node@v6
@@ -63,7 +68,9 @@ jobs:
         run: |
           MOCK_FLAG=""
           if [ "$MOCK_INPUT" = "true" ]; then MOCK_FLAG="--mock"; fi
-          npx tsx cli/experiment.ts -c "${GITHUB_WORKSPACE}/${CONFIG_PATH}" $MOCK_FLAG --skip-missing
+          PREFLIGHT_FLAG=""
+          if [ "$PREFLIGHT_INPUT" = "true" ]; then PREFLIGHT_FLAG="--preflight-only"; fi
+          npx tsx cli/experiment.ts -c "${GITHUB_WORKSPACE}/${CONFIG_PATH}" $MOCK_FLAG $PREFLIGHT_FLAG --skip-missing
       # Runs even when the experiment step fails: run-002 attempt 1 lost
       # ~2500 paid judge calls because the failed step blocked publication.
       # Failed-run data is diagnostic material — pushed on a clearly-labeled

apps/web/cli/experiment.ts

@@ -34,19 +34,21 @@ function parseArgs(argv: string[]) {
   let mock = false;
   let outDir = '';
   let skipMissing = false;
+  let preflightOnly = false;
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     if (a === '-c' || a === '--config') configPath = argv[++i] ?? '';
     else if (a === '--mock') mock = true;
     else if (a === '--out') outDir = argv[++i] ?? '';
     else if (a === '--skip-missing') skipMissing = true;
+    else if (a === '--preflight-only') preflightOnly = true;
     else if (a === '-h' || a === '--help') {
-      process.stdout.write('usage: experiment.ts -c config.json [--mock] [--out DIR] [--skip-missing]\n');
+      process.stdout.write('usage: experiment.ts -c config.json [--mock] [--out DIR] [--skip-missing] [--preflight-only]\n');
       process.exit(0);
     } else fail(`unknown argument: ${a}`);
   }
   if (!configPath) fail('missing -c config.json');
-  return { configPath, mock, outDir, skipMissing };
+  return { configPath, mock, outDir, skipMissing, preflightOnly };
 }
 
 // Real adapter factory. The adapter classes read credentials from fixed env
@@ -142,7 +144,7 @@ function renderReport(summary: ExperimentSummary): string {
 }
 
 async function main() {
-  const { configPath, mock, outDir, skipMissing } = parseArgs(process.argv.slice(2));
+  const { configPath, mock, outDir, skipMissing, preflightOnly } = parseArgs(process.argv.slice(2));
   const configAbs = path.resolve(configPath);
   const configDir = path.dirname(configAbs);
   const rawConfig = JSON.parse(fs.readFileSync(configAbs, 'utf-8'));
@@ -154,7 +156,7 @@ async function main() {
   // run-002 attempt 1 showed a present-but-invalid key burns ~1250 failed
   // calls and sinks the whole run. Mock runs need no credentials.
   let skippedJudges: { id: string; reason: string }[] = [];
-  if (skipMissing && !mock) {
+  if ((skipMissing || preflightOnly) && !mock) {
     const { runnable, skipped } = partitionJudgesByEnv(config.judges, process.env);
     skippedJudges = skipped;
     for (const sk of skipped) {
@@ -163,10 +165,26 @@ async function main() {
     if (runnable.length === 0) fail('all judges skipped — no credentials found; set at least ANTHROPIC_API_KEY or OPENAI_API_KEY');
     const pf = await preflightJudges(runnable, realAdapterFor, (m) => process.stderr.write(m + '\n'));
     skippedJudges = [...skippedJudges, ...pf.skipped];
-    if (pf.runnable.length === 0) fail('all judges failed preflight — no working credentials (check the API keys, base URLs and model ids)');
+    if (pf.runnable.length === 0 && !preflightOnly) fail('all judges failed preflight — no working credentials (check the API keys, base URLs and model ids)');
     config = { ...config, judges: pf.runnable };
   }
 
+  // --preflight-only: credential check mode — a few ~8-token probe calls,
+  // no scoring, no files written. Exit 0 only when EVERY configured judge
+  // passed, so a red workflow run means "fix a secret before spending on a
+  // real run" and a green one means "all three families are ready".
+  if (preflightOnly) {
+    if (mock) fail('--preflight-only is meaningless with --mock (the mock needs no credentials)');
+    process.stderr.write(`\nPREFLIGHT SUMMARY — ${config.judges.length} ok, ${skippedJudges.length} failed/missing\n`);
+    for (const j of config.judges) process.stderr.write(`  ✓ ${j.id}\n`);
+    for (const sk of skippedJudges) process.stderr.write(`  ✗ ${sk.id}: ${sk.reason}\n`);
+    if (skippedJudges.length > 0) {
+      fail(`${skippedJudges.length} judge(s) not ready — fix the secrets above, then re-run the preflight`);
+    }
+    process.stderr.write('\nall judges ready — launch the real run.\n');
+    return;
+  }
+
   const baseOut = outDir ? path.resolve(outDir) : configDir;
   const resultsDir = mock ? path.join(baseOut, 'mock') : path.join(baseOut, 'results');
   fs.mkdirSync(path.join(resultsDir, 'raw'), { recursive: true });

research/experiments/run-002/README.md

@@ -30,16 +30,25 @@ place.
 2. Edit `config.json` here: set the open-weight judge's `"model"` to
    the EXACT model identifier your endpoint serves (the committed value
    is a placeholder).
-3. Actions → **Experiment Run** → Run workflow →
+3. **Credential check first (free, ~2 min):** Actions → **Experiment
+   Run** → Run workflow → config =
+   `research/experiments/run-002/config.json`, mock = false,
+   **preflight_only = true**. One ~8-token probe call per judge, no
+   scoring, no spend. Green run = every judge answered (the log shows
+   `✓` per judge) — the secrets are proven working. Red run = the log
+   names exactly which judge/secret to fix; fix it and re-run the
+   preflight. (Secret values are never displayed by GitHub — the
+   "Updated X ago" timestamp plus this preflight ARE the confirmation.)
+4. Actions → **Experiment Run** → Run workflow →
    config = `research/experiments/run-002/config.json`, mock = false.
-4. Optional dry run first: same config with **mock = true** (free,
+5. Optional dry run first: same config with **mock = true** (free,
    minutes, validates the pipeline end-to-end; mock output is never a
    measurement — in particular the mock judge does not recognize the
    corpus's adversarial items, so a mock dry run reports mass bound
    "failures"; that is expected noise, not a finding). This exact
    mock run was executed locally during Phase B integration:
    3 750/3 750 calls, 750 cells, all 3 judges, zero pipeline failures.
-5. The workflow pushes a results branch and opens (or links) the
+6. The workflow pushes a results branch and opens (or links) the
    results PR. Review REPORT.md and the raw JSONL, then merge — that
    merge is the publication of the run, including any bound failures.