fix(science): external-audit response wave 1 — honest power analysis, earned L3, nominal α

apps/web/app/api/score/route.ts

@@ -50,6 +50,7 @@ const ScoreRequestSchema = z.object({
 function toDetSummary(check: Awaited<ReturnType<typeof verifyCitations>> | null) {
   return check ? {
     effective: verificationEffective(check),
+    verified: check.totals.verified,
     notFound: check.totals.notFound,
     truncated: check.totals.truncated,
   } : undefined;

apps/web/cli/piga.ts

@@ -40,6 +40,10 @@ interface CliArgs {
   file?: string;
   subjectModel?: string;
   judgeModel?: string;
+  /** Provider for the JUDGE adapter, independent of the subject's —
+   *  without this, subject and judge share one provider and can even be
+   *  the same model (self-judging bias, external audit finding). */
+  judgeProvider?: 'anthropic' | 'openai';
   output?: string;
   help: boolean;
 }
@@ -52,6 +56,7 @@ function parseArgs(argv: string[]): CliArgs {
       case '-f': case '--file': args.file = argv[++i]; break;
       case '--subject-model': args.subjectModel = argv[++i]; break;
       case '--judge-model': args.judgeModel = argv[++i]; break;
+      case '--judge-provider': args.judgeProvider = argv[++i] as 'anthropic' | 'openai'; break;
       case '-o': case '--output': args.output = argv[++i]; break;
       case '-h': case '--help': args.help = true; break;
       default:
@@ -60,6 +65,10 @@ function parseArgs(argv: string[]): CliArgs {
     }
     i++;
   }
+  if (args.judgeProvider && !['anthropic', 'openai'].includes(args.judgeProvider)) {
+    console.error(`--judge-provider must be anthropic or openai (got ${args.judgeProvider})`);
+    process.exit(1);
+  }
   return args;
 }
 
@@ -88,6 +97,9 @@ OPTIONS:
   -f, --file <path>        PIGA dataset (required)
   --subject-model <model>  Model under test (default: provider default)
   --judge-model <model>    Classifier judge (default: CAIMS_SCORING_MODEL or provider default)
+  --judge-provider <p>     Judge provider (anthropic|openai), independent of the
+                           subject's — use it so subject and judge are never the
+                           same family (self-judging bias)
   -o, --output <path>      Write full JSON results
   -h, --help               This help
 
@@ -105,13 +117,18 @@ see docs/piga-spec-a7.md. v0 is a prototype: no validity evidence yet.`);
   }
 
   const provider = getProviderFromEnv();
-  const defaultModel = provider === 'openai' ? 'gpt-4o' : 'claude-sonnet-4-20250514';
-  const subjectModel = args.subjectModel || defaultModel;
-  const judgeModel = args.judgeModel || process.env.CAIMS_SCORING_MODEL || defaultModel;
+  const judgeProvider = args.judgeProvider || provider;
+  const defaultModelFor = (p: string) => (p === 'openai' ? 'gpt-4o' : 'claude-sonnet-4-20250514');
+  const subjectModel = args.subjectModel || defaultModelFor(provider);
+  const judgeModel = args.judgeModel || process.env.CAIMS_SCORING_MODEL || defaultModelFor(judgeProvider);
   const adapter = getAdapter();
+  const judgeAdapter = args.judgeProvider ? getAdapter(args.judgeProvider) : adapter;
 
   process.stderr.write(`CAIMS-PIGA ${PIGA_PROTOCOL_VERSION} — ${dataset.name}\n`);
-  process.stderr.write(`subject: ${subjectModel} | judge: ${judgeModel} (prompt ${PIGA_PROMPT_HASH}) | items: ${dataset.items.length}\n\n`);
+  process.stderr.write(`subject: ${provider}/${subjectModel} | judge: ${judgeProvider}/${judgeModel} (prompt ${PIGA_PROMPT_HASH}) | items: ${dataset.items.length}\n\n`);
+  if (provider === judgeProvider) {
+    process.stderr.write(`WARNING: subject and judge share the ${provider} provider family${subjectModel === judgeModel ? ' AND the same model (self-judging)' : ''} — classifications are confounded by family style; use --judge-provider for a cross-family run.\n\n`);
+  }
 
   const results: PigaItemResult[] = [];
   for (let i = 0; i < dataset.items.length; i++) {
@@ -124,7 +141,7 @@ see docs/piga-spec-a7.md. v0 is a prototype: no validity evidence yet.`);
       );
       const subjectTruncated = subject.outputTokens >= SUBJECT_MAX_TOKENS;
       const truncNote = subjectTruncated ? ' [TRUNCATED at cap]' : '';
-      const judged = await classifyAndScorePiga({ item, response: subject.content, model: judgeModel });
+      const judged = await classifyAndScorePiga({ item, response: subject.content, model: judgeModel, adapter: judgeAdapter });
       if (!judged) {
         process.stderr.write(` JUDGE FAILED${truncNote}\n`);
         results.push({ id: item.id, clarification_expectation: item.clarification_expectation, harm_if_wrong: item.harm_if_wrong, subjectResponse: subject.content, subjectOutputTokens: subject.outputTokens, subjectTruncated, classification: null, score: null, error: 'judge classification failed' });

apps/web/cli/power-analysis.ts

@@ -68,11 +68,21 @@ console.log(`POWER ANALYSIS — parameters from ${summary.runId} (protocol ${sum
 console.log(`within-cell composite SDs: median ${medianSd.toFixed(2)}, max ${maxSd.toFixed(2)} (${cellSds.length} cells)`);
 console.log(`observed |inter-judge diff| per item: ${observedDiffs.map(d => d.toFixed(1)).join(', ')}\n`);
 
-// ── A. Samples per control cell: power to flag a true bound violation ──────
-console.log('A. SAMPLES PER CONTROL CELL — P(flag a true violation of Δ points | within-cell SD σ)');
-console.log('   (Δ<0 rows would give false-alarm rates; H1 rule: mean of n samples > bound)');
+// ── A. Samples per control cell: P(flag) of the PREREGISTERED H1 rule ──────
+// HONESTY HEADER (external audit finding, accepted): the H1 rule
+// "mean of n samples > bound" is a preregistered DECISION RULE, not an
+// alpha-controlled hypothesis test. At the exact boundary (Δ=0) it flags
+// ~50% of truly-borderline cells by construction. The table therefore
+// reports P(flag | Δ) for the rule as preregistered — including the
+// false-alarm rows Δ<0 — and table A2 gives the power of a PROPER
+// one-sided alpha=0.05 z-test for comparison. Under that stricter test,
+// n=5 at the adversarial-worst sigma has power ~0.43 for Δ=5, and ~15
+// samples are needed for 80% power.
+console.log('A. SAMPLES PER CONTROL CELL — P(flag | Δ) of the preregistered H1 rule (mean of n samples > bound)');
+console.log('   NOT an alpha-controlled test: at Δ=0 the rule flags ~50% by construction.');
+console.log('   Δ>0 rows = detection rates for true violations; Δ<0 rows = FALSE-ALARM rates for truly-passing cells.');
 const sigmas = [medianSd, 5, maxSd]; // simulate at the UNROUNDED observed values
-const deltas = [2, 5, 10];
+const deltas = [-5, -2, 2, 5, 10];
 const ns = [3, 5, 10, 15, 25];
 console.log('   σ\\n        ' + ns.map(n => `n=${n}`.padStart(7)).join(''));
 let seedCounter = SEED;
@@ -81,10 +91,35 @@ for (const sigma of sigmas) {
     const row = ns.map(n =>
       boundDetectionPower({ sigma, delta, n, sims: SIMS, seed: seedCounter++ }).toFixed(3).padStart(7)
     );
-    console.log(`   σ=${sigma.toFixed(2).padEnd(5)} Δ=${String(delta).padEnd(3)}` + row.join(''));
+    console.log(`   σ=${sigma.toFixed(2).padEnd(5)} Δ=${String(delta).padEnd(4)}` + row.join(''));
   }
 }
 
+// ── A2. Power of a PROPER one-sided alpha=0.05 test (analytic) ─────────────
+// power = Phi(Δ·sqrt(n)/σ − z_{0.95}), z_{0.95} = 1.6449. Analytic, no
+// simulation needed; assumes known σ (optimistic — a t-test with estimated
+// σ at n=5 is weaker still).
+const Z95 = 1.6449;
+const phi = (x: number) => 0.5 * (1 + erf(x / Math.SQRT2));
+function erf(x: number): number {
+  // Abramowitz–Stegun 7.1.26, |error| < 1.5e-7 — ample for 3 decimals.
+  const sign = x < 0 ? -1 : 1;
+  const t = 1 / (1 + 0.3275911 * Math.abs(x));
+  const y = 1 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.exp(-x * x);
+  return sign * y;
+}
+console.log('\nA2. SAME CELLS UNDER A ONE-SIDED alpha=0.05 z-TEST — power = Phi(Δ·sqrt(n)/σ − 1.6449)');
+console.log('   (known-σ assumption: OPTIMISTIC; the t-test power at small n is lower)');
+console.log('   σ\\n        ' + ns.map(n => `n=${n}`.padStart(7)).join(''));
+for (const sigma of sigmas) {
+  for (const delta of [2, 5, 10]) {
+    const row = ns.map(n => phi(delta * Math.sqrt(n) / sigma - Z95).toFixed(3).padStart(7));
+    console.log(`   σ=${sigma.toFixed(2).padEnd(5)} Δ=${String(delta).padEnd(4)}` + row.join(''));
+  }
+}
+const nFor80 = (sigma: number, delta: number) => Math.ceil(((0.8416 + Z95) * sigma / delta) ** 2);
+console.log(`   n for 80% power at Δ=5: σ=${medianSd.toFixed(2)} → ${nFor80(medianSd, 5)}; σ=5.00 → ${nFor80(5, 5)}; σ=${maxSd.toFixed(2)} → ${nFor80(maxSd, 5)}`);
+
 // ── B. Items for mean |judge diff| precision (bootstrap from observed) ─────
 console.log('\nB. ITEMS FOR MEAN |JUDGE DIFF| PRECISION — bootstrap 95% CI half-width (pts)');
 const itemCounts = [10, 25, 50, 100, 200];

apps/web/lib/__tests__/api-score-ensemble.test.ts

@@ -148,27 +148,60 @@ describe('POST /api/score — ensemble & n-sample (v2.1)', () => {
     expect(body.data.evidenceCard.profile.cq.n).toBe(2);
   });
 
-  it('verifyCitations on the ensemble path: runs the verifier, attaches results, lifts L2→L3', async () => {
+  it('verifyCitations on the ensemble path: a VERIFIED reference earns the L2→L3 lift', async () => {
     process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b';
     mockJudge
       .mockResolvedValueOnce(rubricJson(60))
       .mockResolvedValueOnce(rubricJson(80));
     mockVerifyCitations.mockResolvedValueOnce({
       ran: true,
-      citations: [{ kind: 'doi', raw: '10.1/x', id: '10.1/x', status: 'not_found' }],
-      totals: { total: 1, extractedTotal: 1, truncated: false, verified: 0, notFound: 1, unverifiable: 0, networkErrors: 0 },
+      citations: [{ kind: 'doi', raw: '10.1/x', id: '10.1/x', status: 'verified' }],
+      totals: { total: 1, extractedTotal: 1, truncated: false, verified: 1, notFound: 0, unverifiable: 0, networkErrors: 0 },
       note: 'stub',
     });
     const res = await handler(makeRequest({ ...BASE_BODY, ensemble: true, verifyCitations: true }));
     const body = await res.json();
     expect(res.status).toBe(200);
     expect(mockVerifyCitations).toHaveBeenCalledWith(BASE_BODY.response);
     expect(body.data.evidenceCard.evidenceLevel).toBe('L3');
+    expect(body.data.verification.citations.totals.verified).toBe(1);
+  });
+
+  it('a detected fabrication BLOCKS the lift (stays L2) and is a caveat ON THE CARD', async () => {
+    process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b';
+    mockJudge
+      .mockResolvedValueOnce(rubricJson(60))
+      .mockResolvedValueOnce(rubricJson(80));
+    mockVerifyCitations.mockResolvedValueOnce({
+      ran: true,
+      citations: [{ kind: 'doi', raw: '10.1/x', id: '10.1/x', status: 'not_found' }],
+      totals: { total: 1, extractedTotal: 1, truncated: false, verified: 0, notFound: 1, unverifiable: 0, networkErrors: 0 },
+      note: 'stub',
+    });
+    const res = await handler(makeRequest({ ...BASE_BODY, ensemble: true, verifyCitations: true }));
+    const body = await res.json();
+    expect(res.status).toBe(200);
+    expect(body.data.evidenceCard.evidenceLevel).toBe('L2'); // a fake reference can never wear the L3 badge
     expect(body.data.verification.citations.totals.notFound).toBe(1);
-    // detected fabrication is a caveat ON THE CARD, not just a sibling payload
     expect(body.data.evidenceCard.caveats.some((c: string) => c.includes('NON-EXISTENT'))).toBe(true);
   });
 
+  it('a citation-free response can NOT reach L3 — nothing was verified', async () => {
+    process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b';
+    mockJudge
+      .mockResolvedValueOnce(rubricJson(60))
+      .mockResolvedValueOnce(rubricJson(80));
+    mockVerifyCitations.mockResolvedValueOnce({
+      ran: true, citations: [],
+      totals: { total: 0, extractedTotal: 0, truncated: false, verified: 0, notFound: 0, unverifiable: 0, networkErrors: 0 },
+      note: 'stub',
+    });
+    const res = await handler(makeRequest({ ...BASE_BODY, ensemble: true, verifyCitations: true }));
+    const body = await res.json();
+    expect(body.data.evidenceCard.evidenceLevel).toBe('L2');
+    expect(body.data.evidenceCard.caveats.some((c: string) => c.includes('positively verified no reference'))).toBe(true);
+  });
+
   it('an all-network-error verification run does NOT lift the level (stays L2, with caveat)', async () => {
     process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b';
     mockJudge

apps/web/lib/scorers/__tests__/evidence-card.test.ts

@@ -118,35 +118,45 @@ describe('buildEvidenceCardFromEnsemble', () => {
     expect(card.caveats.some(c => c.includes('failed entirely'))).toBe(true);
   });
 
-  it('an EFFECTIVE deterministic run lifts L2 to L3 — and never lifts L1', () => {
+  it('L3 must be EARNED: effective run with >=1 verified reference and 0 fabrications lifts L2 — never L1', () => {
     const a = judge({ id: 'anthropic:a', provider: 'anthropic' });
     const b = judge({ id: 'openai:b', provider: 'openai' });
-    const det = { effective: true, notFound: 0, truncated: false };
+    const det = { effective: true, verified: 2, notFound: 0, truncated: false };
     expect(buildEvidenceCardFromEnsemble(ensembleResult([a, b]), { deterministicChecks: det }).evidenceLevel).toBe('L3');
     expect(buildEvidenceCardFromEnsemble(ensembleResult([a]), { deterministicChecks: det }).evidenceLevel).toBe('L1');
   });
 
+  it('a citation-free response can NOT reach L3 (nothing was verified), with the reason in a caveat', () => {
+    const a = judge({ id: 'anthropic:a', provider: 'anthropic' });
+    const b = judge({ id: 'openai:b', provider: 'openai' });
+    const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]),
+      { deterministicChecks: { effective: true, verified: 0, notFound: 0, truncated: false } });
+    expect(card.evidenceLevel).toBe('L2');
+    expect(card.caveats.some(c => c.includes('positively verified no reference'))).toBe(true);
+  });
+
   it('an INEFFECTIVE run (all network errors / truncated) never lifts, and says so in a caveat', () => {
     const a = judge({ id: 'anthropic:a', provider: 'anthropic' });
     const b = judge({ id: 'openai:b', provider: 'openai' });
     const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]),
-      { deterministicChecks: { effective: false, notFound: 0, truncated: false } });
+      { deterministicChecks: { effective: false, verified: 0, notFound: 0, truncated: false } });
     expect(card.evidenceLevel).toBe('L2');
     expect(card.caveats.some(c => c.includes('established nothing'))).toBe(true);
   });
 
-  it('detected fabrications become a caveat ON THE CARD — the alarm is never buried again', () => {
+  it('detected fabrications BLOCK the lift and become a caveat — a fake reference can never wear the L3 badge', () => {
     const a = judge({ id: 'anthropic:a', provider: 'anthropic' });
     const b = judge({ id: 'openai:b', provider: 'openai' });
     const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]),
-      { deterministicChecks: { effective: true, notFound: 2, truncated: false } });
-    expect(card.evidenceLevel).toBe('L3');
+      { deterministicChecks: { effective: true, verified: 3, notFound: 2, truncated: false } });
+    expect(card.evidenceLevel).toBe('L2');
     expect(card.caveats.some(c => c.includes('2 NON-EXISTENT reference'))).toBe(true);
+    expect(card.caveats.some(c => c.includes('NOT lifted'))).toBe(true);
   });
 
   it('truncation adds its own caveat', () => {
     const card = buildEvidenceCardFromEnsemble(ensembleResult([judge({})]),
-      { deterministicChecks: { effective: false, notFound: 0, truncated: true } });
+      { deterministicChecks: { effective: false, verified: 0, notFound: 0, truncated: true } });
     expect(card.caveats.some(c => c.includes('truncated'))).toBe(true);
   });
 });

apps/web/lib/scorers/evidence-card.ts

@@ -80,16 +80,28 @@ function registryRef(): string {
 export interface DeterministicChecksSummary {
   /** True only when the run actually established facts (see verificationEffective). */
   effective: boolean;
+  /** References positively confirmed to exist in a public registry. */
+  verified: number;
   notFound: number;
   truncated: boolean;
 }
 
+// L3 is a badge readers treat as "facts were checked and held up", so it
+// must be EARNED, never inherited from an empty check (external audit
+// finding): the lift requires at least one positively verified reference
+// AND zero non-existent ones. A citation-free response, an all-network-error
+// run, or a text carrying a fabricated reference all stay at L2 with the
+// reason in the caveats.
+export function deterministicLiftEarned(det: DeterministicChecksSummary | undefined): boolean {
+  return !!det && det.effective && det.verified > 0 && det.notFound === 0;
+}
+
 function verificationCaveats(det: DeterministicChecksSummary | undefined): string[] {
   if (!det) return [];
   const out: string[] = [];
   if (det.notFound > 0) {
     out.push(
-      `deterministic citation verification found ${det.notFound} NON-EXISTENT reference(s) — see verification.citations before trusting this profile`
+      `deterministic citation verification found ${det.notFound} NON-EXISTENT reference(s) — see verification.citations before trusting this profile; the evidence level was NOT lifted`
     );
   }
   if (det.truncated) {
@@ -98,6 +110,9 @@ function verificationCaveats(det: DeterministicChecksSummary | undefined): strin
   if (!det.effective) {
     out.push('deterministic checks ran but established nothing (network errors or truncation) — the evidence level was NOT lifted');
   }
+  if (det.effective && det.verified === 0 && det.notFound === 0) {
+    out.push('verification ran but positively verified no reference (none present, or none carried a checkable identifier) — the evidence level was NOT lifted');
+  }
   return out;
 }
 
@@ -142,9 +157,10 @@ export function buildEvidenceCardFromSingle(
  *
  * Level is computed: L2 requires ≥2 DISTINCT PROVIDER FAMILIES among the ok
  * judges (two models of one family do not make an ensemble of families);
- * the L2→L3 lift requires a deterministic-verification run that was
- * EFFECTIVE (established facts: not truncated, not all-network-error) —
- * a run that verified nothing must not upgrade the label.
+ * the L2→L3 lift must be EARNED (deterministicLiftEarned): an effective
+ * verification run with ≥1 positively verified reference and 0 fabricated
+ * ones. A run that verified nothing — or that FOUND a fabrication — must
+ * not upgrade the label.
  */
 export function buildEvidenceCardFromEnsemble(
   result: EnsembleScores,
@@ -182,7 +198,7 @@ export function buildEvidenceCardFromEnsemble(
 
   const distinctProviders = new Set(judges.map(j => j.provider)).size;
   let level: EvidenceLevel = distinctProviders >= 2 ? 'L2' : 'L1';
-  if (level === 'L2' && opts.deterministicChecks?.effective) level = 'L3';
+  if (level === 'L2' && deterministicLiftEarned(opts.deterministicChecks)) level = 'L3';
 
   const caveats: string[] = [...STANDING_CAVEATS, ...verificationCaveats(opts.deterministicChecks)];
   if (result.failedJudges.length > 0) {