diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index e3a152c..f118072 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -23,6 +23,10 @@ on: description: "Run with the deterministic mock judge (pipeline validation only)" type: boolean default: false + preflight_only: + description: "Credential check only: one tiny probe call per judge, no scoring, no spend. Green = all judges ready; red = a secret needs fixing." + type: boolean + default: false permissions: contents: write @@ -40,6 +44,7 @@ jobs: # scripts in a secret-holding workflow. CONFIG_PATH: ${{ inputs.config }} MOCK_INPUT: ${{ inputs.mock }} + PREFLIGHT_INPUT: ${{ inputs.preflight_only }} steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v6 @@ -63,7 +68,9 @@ jobs: run: | MOCK_FLAG="" if [ "$MOCK_INPUT" = "true" ]; then MOCK_FLAG="--mock"; fi - npx tsx cli/experiment.ts -c "${GITHUB_WORKSPACE}/${CONFIG_PATH}" $MOCK_FLAG --skip-missing + PREFLIGHT_FLAG="" + if [ "$PREFLIGHT_INPUT" = "true" ]; then PREFLIGHT_FLAG="--preflight-only"; fi + npx tsx cli/experiment.ts -c "${GITHUB_WORKSPACE}/${CONFIG_PATH}" $MOCK_FLAG $PREFLIGHT_FLAG --skip-missing # Runs even when the experiment step fails: run-002 attempt 1 lost # ~2500 paid judge calls because the failed step blocked publication. # Failed-run data is diagnostic material — pushed on a clearly-labeled diff --git a/apps/web/cli/experiment.ts b/apps/web/cli/experiment.ts index e80f0f0..b9b69b9 100644 --- a/apps/web/cli/experiment.ts +++ b/apps/web/cli/experiment.ts @@ -34,19 +34,21 @@ function parseArgs(argv: string[]) { let mock = false; let outDir = ''; let skipMissing = false; + let preflightOnly = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '-c' || a === '--config') configPath = argv[++i] ?? ''; else if (a === '--mock') mock = true; else if (a === '--out') outDir = argv[++i] ?? ''; else if (a === '--skip-missing') skipMissing = true; + else if (a === '--preflight-only') preflightOnly = true; else if (a === '-h' || a === '--help') { - process.stdout.write('usage: experiment.ts -c config.json [--mock] [--out DIR] [--skip-missing]\n'); + process.stdout.write('usage: experiment.ts -c config.json [--mock] [--out DIR] [--skip-missing] [--preflight-only]\n'); process.exit(0); } else fail(`unknown argument: ${a}`); } if (!configPath) fail('missing -c config.json'); - return { configPath, mock, outDir, skipMissing }; + return { configPath, mock, outDir, skipMissing, preflightOnly }; } // Real adapter factory. The adapter classes read credentials from fixed env @@ -142,7 +144,7 @@ function renderReport(summary: ExperimentSummary): string { } async function main() { - const { configPath, mock, outDir, skipMissing } = parseArgs(process.argv.slice(2)); + const { configPath, mock, outDir, skipMissing, preflightOnly } = parseArgs(process.argv.slice(2)); const configAbs = path.resolve(configPath); const configDir = path.dirname(configAbs); const rawConfig = JSON.parse(fs.readFileSync(configAbs, 'utf-8')); @@ -154,7 +156,7 @@ async function main() { // run-002 attempt 1 showed a present-but-invalid key burns ~1250 failed // calls and sinks the whole run. Mock runs need no credentials. let skippedJudges: { id: string; reason: string }[] = []; - if (skipMissing && !mock) { + if ((skipMissing || preflightOnly) && !mock) { const { runnable, skipped } = partitionJudgesByEnv(config.judges, process.env); skippedJudges = skipped; for (const sk of skipped) { @@ -163,10 +165,26 @@ async function main() { if (runnable.length === 0) fail('all judges skipped — no credentials found; set at least ANTHROPIC_API_KEY or OPENAI_API_KEY'); const pf = await preflightJudges(runnable, realAdapterFor, (m) => process.stderr.write(m + '\n')); skippedJudges = [...skippedJudges, ...pf.skipped]; - if (pf.runnable.length === 0) fail('all judges failed preflight — no working credentials (check the API keys, base URLs and model ids)'); + if (pf.runnable.length === 0 && !preflightOnly) fail('all judges failed preflight — no working credentials (check the API keys, base URLs and model ids)'); config = { ...config, judges: pf.runnable }; } + // --preflight-only: credential check mode — a few ~8-token probe calls, + // no scoring, no files written. Exit 0 only when EVERY configured judge + // passed, so a red workflow run means "fix a secret before spending on a + // real run" and a green one means "all three families are ready". + if (preflightOnly) { + if (mock) fail('--preflight-only is meaningless with --mock (the mock needs no credentials)'); + process.stderr.write(`\nPREFLIGHT SUMMARY — ${config.judges.length} ok, ${skippedJudges.length} failed/missing\n`); + for (const j of config.judges) process.stderr.write(` ✓ ${j.id}\n`); + for (const sk of skippedJudges) process.stderr.write(` ✗ ${sk.id}: ${sk.reason}\n`); + if (skippedJudges.length > 0) { + fail(`${skippedJudges.length} judge(s) not ready — fix the secrets above, then re-run the preflight`); + } + process.stderr.write('\nall judges ready — launch the real run.\n'); + return; + } + const baseOut = outDir ? path.resolve(outDir) : configDir; const resultsDir = mock ? path.join(baseOut, 'mock') : path.join(baseOut, 'results'); fs.mkdirSync(path.join(resultsDir, 'raw'), { recursive: true }); diff --git a/research/experiments/run-002/README.md b/research/experiments/run-002/README.md index 412246f..d553557 100644 --- a/research/experiments/run-002/README.md +++ b/research/experiments/run-002/README.md @@ -30,16 +30,25 @@ place. 2. Edit `config.json` here: set the open-weight judge's `"model"` to the EXACT model identifier your endpoint serves (the committed value is a placeholder). -3. Actions → **Experiment Run** → Run workflow → +3. **Credential check first (free, ~2 min):** Actions → **Experiment + Run** → Run workflow → config = + `research/experiments/run-002/config.json`, mock = false, + **preflight_only = true**. One ~8-token probe call per judge, no + scoring, no spend. Green run = every judge answered (the log shows + `✓` per judge) — the secrets are proven working. Red run = the log + names exactly which judge/secret to fix; fix it and re-run the + preflight. (Secret values are never displayed by GitHub — the + "Updated X ago" timestamp plus this preflight ARE the confirmation.) +4. Actions → **Experiment Run** → Run workflow → config = `research/experiments/run-002/config.json`, mock = false. -4. Optional dry run first: same config with **mock = true** (free, +5. Optional dry run first: same config with **mock = true** (free, minutes, validates the pipeline end-to-end; mock output is never a measurement — in particular the mock judge does not recognize the corpus's adversarial items, so a mock dry run reports mass bound "failures"; that is expected noise, not a finding). This exact mock run was executed locally during Phase B integration: 3 750/3 750 calls, 750 cells, all 3 judges, zero pipeline failures. -5. The workflow pushes a results branch and opens (or links) the +6. The workflow pushes a results branch and opens (or links) the results PR. Review REPORT.md and the raw JSONL, then merge — that merge is the publication of the run, including any bound failures.