diff --git a/apps/web/app/api/score/route.ts b/apps/web/app/api/score/route.ts index 8a8ad20..99e952a 100644 --- a/apps/web/app/api/score/route.ts +++ b/apps/web/app/api/score/route.ts @@ -50,6 +50,7 @@ const ScoreRequestSchema = z.object({ function toDetSummary(check: Awaited> | null) { return check ? { effective: verificationEffective(check), + verified: check.totals.verified, notFound: check.totals.notFound, truncated: check.totals.truncated, } : undefined; diff --git a/apps/web/cli/piga.ts b/apps/web/cli/piga.ts index 80bf74f..6331bc1 100644 --- a/apps/web/cli/piga.ts +++ b/apps/web/cli/piga.ts @@ -40,6 +40,10 @@ interface CliArgs { file?: string; subjectModel?: string; judgeModel?: string; + /** Provider for the JUDGE adapter, independent of the subject's — + * without this, subject and judge share one provider and can even be + * the same model (self-judging bias, external audit finding). */ + judgeProvider?: 'anthropic' | 'openai'; output?: string; help: boolean; } @@ -52,6 +56,7 @@ function parseArgs(argv: string[]): CliArgs { case '-f': case '--file': args.file = argv[++i]; break; case '--subject-model': args.subjectModel = argv[++i]; break; case '--judge-model': args.judgeModel = argv[++i]; break; + case '--judge-provider': args.judgeProvider = argv[++i] as 'anthropic' | 'openai'; break; case '-o': case '--output': args.output = argv[++i]; break; case '-h': case '--help': args.help = true; break; default: @@ -60,6 +65,10 @@ function parseArgs(argv: string[]): CliArgs { } i++; } + if (args.judgeProvider && !['anthropic', 'openai'].includes(args.judgeProvider)) { + console.error(`--judge-provider must be anthropic or openai (got ${args.judgeProvider})`); + process.exit(1); + } return args; } @@ -88,6 +97,9 @@ OPTIONS: -f, --file PIGA dataset (required) --subject-model Model under test (default: provider default) --judge-model Classifier judge (default: CAIMS_SCORING_MODEL or provider default) + --judge-provider

Judge provider (anthropic|openai), independent of the + subject's — use it so subject and judge are never the + same family (self-judging bias) -o, --output Write full JSON results -h, --help This help @@ -105,13 +117,18 @@ see docs/piga-spec-a7.md. v0 is a prototype: no validity evidence yet.`); } const provider = getProviderFromEnv(); - const defaultModel = provider === 'openai' ? 'gpt-4o' : 'claude-sonnet-4-20250514'; - const subjectModel = args.subjectModel || defaultModel; - const judgeModel = args.judgeModel || process.env.CAIMS_SCORING_MODEL || defaultModel; + const judgeProvider = args.judgeProvider || provider; + const defaultModelFor = (p: string) => (p === 'openai' ? 'gpt-4o' : 'claude-sonnet-4-20250514'); + const subjectModel = args.subjectModel || defaultModelFor(provider); + const judgeModel = args.judgeModel || process.env.CAIMS_SCORING_MODEL || defaultModelFor(judgeProvider); const adapter = getAdapter(); + const judgeAdapter = args.judgeProvider ? getAdapter(args.judgeProvider) : adapter; process.stderr.write(`CAIMS-PIGA ${PIGA_PROTOCOL_VERSION} — ${dataset.name}\n`); - process.stderr.write(`subject: ${subjectModel} | judge: ${judgeModel} (prompt ${PIGA_PROMPT_HASH}) | items: ${dataset.items.length}\n\n`); + process.stderr.write(`subject: ${provider}/${subjectModel} | judge: ${judgeProvider}/${judgeModel} (prompt ${PIGA_PROMPT_HASH}) | items: ${dataset.items.length}\n\n`); + if (provider === judgeProvider) { + process.stderr.write(`WARNING: subject and judge share the ${provider} provider family${subjectModel === judgeModel ? ' AND the same model (self-judging)' : ''} — classifications are confounded by family style; use --judge-provider for a cross-family run.\n\n`); + } const results: PigaItemResult[] = []; for (let i = 0; i < dataset.items.length; i++) { @@ -124,7 +141,7 @@ see docs/piga-spec-a7.md. v0 is a prototype: no validity evidence yet.`); ); const subjectTruncated = subject.outputTokens >= SUBJECT_MAX_TOKENS; const truncNote = subjectTruncated ? ' [TRUNCATED at cap]' : ''; - const judged = await classifyAndScorePiga({ item, response: subject.content, model: judgeModel }); + const judged = await classifyAndScorePiga({ item, response: subject.content, model: judgeModel, adapter: judgeAdapter }); if (!judged) { process.stderr.write(` JUDGE FAILED${truncNote}\n`); results.push({ id: item.id, clarification_expectation: item.clarification_expectation, harm_if_wrong: item.harm_if_wrong, subjectResponse: subject.content, subjectOutputTokens: subject.outputTokens, subjectTruncated, classification: null, score: null, error: 'judge classification failed' }); diff --git a/apps/web/cli/power-analysis.ts b/apps/web/cli/power-analysis.ts index a99eb11..d11a8bf 100644 --- a/apps/web/cli/power-analysis.ts +++ b/apps/web/cli/power-analysis.ts @@ -68,11 +68,21 @@ console.log(`POWER ANALYSIS — parameters from ${summary.runId} (protocol ${sum console.log(`within-cell composite SDs: median ${medianSd.toFixed(2)}, max ${maxSd.toFixed(2)} (${cellSds.length} cells)`); console.log(`observed |inter-judge diff| per item: ${observedDiffs.map(d => d.toFixed(1)).join(', ')}\n`); -// ── A. Samples per control cell: power to flag a true bound violation ────── -console.log('A. SAMPLES PER CONTROL CELL — P(flag a true violation of Δ points | within-cell SD σ)'); -console.log(' (Δ<0 rows would give false-alarm rates; H1 rule: mean of n samples > bound)'); +// ── A. Samples per control cell: P(flag) of the PREREGISTERED H1 rule ────── +// HONESTY HEADER (external audit finding, accepted): the H1 rule +// "mean of n samples > bound" is a preregistered DECISION RULE, not an +// alpha-controlled hypothesis test. At the exact boundary (Δ=0) it flags +// ~50% of truly-borderline cells by construction. The table therefore +// reports P(flag | Δ) for the rule as preregistered — including the +// false-alarm rows Δ<0 — and table A2 gives the power of a PROPER +// one-sided alpha=0.05 z-test for comparison. Under that stricter test, +// n=5 at the adversarial-worst sigma has power ~0.43 for Δ=5, and ~15 +// samples are needed for 80% power. +console.log('A. SAMPLES PER CONTROL CELL — P(flag | Δ) of the preregistered H1 rule (mean of n samples > bound)'); +console.log(' NOT an alpha-controlled test: at Δ=0 the rule flags ~50% by construction.'); +console.log(' Δ>0 rows = detection rates for true violations; Δ<0 rows = FALSE-ALARM rates for truly-passing cells.'); const sigmas = [medianSd, 5, maxSd]; // simulate at the UNROUNDED observed values -const deltas = [2, 5, 10]; +const deltas = [-5, -2, 2, 5, 10]; const ns = [3, 5, 10, 15, 25]; console.log(' σ\\n ' + ns.map(n => `n=${n}`.padStart(7)).join('')); let seedCounter = SEED; @@ -81,10 +91,35 @@ for (const sigma of sigmas) { const row = ns.map(n => boundDetectionPower({ sigma, delta, n, sims: SIMS, seed: seedCounter++ }).toFixed(3).padStart(7) ); - console.log(` σ=${sigma.toFixed(2).padEnd(5)} Δ=${String(delta).padEnd(3)}` + row.join('')); + console.log(` σ=${sigma.toFixed(2).padEnd(5)} Δ=${String(delta).padEnd(4)}` + row.join('')); } } +// ── A2. Power of a PROPER one-sided alpha=0.05 test (analytic) ───────────── +// power = Phi(Δ·sqrt(n)/σ − z_{0.95}), z_{0.95} = 1.6449. Analytic, no +// simulation needed; assumes known σ (optimistic — a t-test with estimated +// σ at n=5 is weaker still). +const Z95 = 1.6449; +const phi = (x: number) => 0.5 * (1 + erf(x / Math.SQRT2)); +function erf(x: number): number { + // Abramowitz–Stegun 7.1.26, |error| < 1.5e-7 — ample for 3 decimals. + const sign = x < 0 ? -1 : 1; + const t = 1 / (1 + 0.3275911 * Math.abs(x)); + const y = 1 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.exp(-x * x); + return sign * y; +} +console.log('\nA2. SAME CELLS UNDER A ONE-SIDED alpha=0.05 z-TEST — power = Phi(Δ·sqrt(n)/σ − 1.6449)'); +console.log(' (known-σ assumption: OPTIMISTIC; the t-test power at small n is lower)'); +console.log(' σ\\n ' + ns.map(n => `n=${n}`.padStart(7)).join('')); +for (const sigma of sigmas) { + for (const delta of [2, 5, 10]) { + const row = ns.map(n => phi(delta * Math.sqrt(n) / sigma - Z95).toFixed(3).padStart(7)); + console.log(` σ=${sigma.toFixed(2).padEnd(5)} Δ=${String(delta).padEnd(4)}` + row.join('')); + } +} +const nFor80 = (sigma: number, delta: number) => Math.ceil(((0.8416 + Z95) * sigma / delta) ** 2); +console.log(` n for 80% power at Δ=5: σ=${medianSd.toFixed(2)} → ${nFor80(medianSd, 5)}; σ=5.00 → ${nFor80(5, 5)}; σ=${maxSd.toFixed(2)} → ${nFor80(maxSd, 5)}`); + // ── B. Items for mean |judge diff| precision (bootstrap from observed) ───── console.log('\nB. ITEMS FOR MEAN |JUDGE DIFF| PRECISION — bootstrap 95% CI half-width (pts)'); const itemCounts = [10, 25, 50, 100, 200]; diff --git a/apps/web/lib/__tests__/api-score-ensemble.test.ts b/apps/web/lib/__tests__/api-score-ensemble.test.ts index 742e481..8a0698e 100644 --- a/apps/web/lib/__tests__/api-score-ensemble.test.ts +++ b/apps/web/lib/__tests__/api-score-ensemble.test.ts @@ -148,15 +148,15 @@ describe('POST /api/score — ensemble & n-sample (v2.1)', () => { expect(body.data.evidenceCard.profile.cq.n).toBe(2); }); - it('verifyCitations on the ensemble path: runs the verifier, attaches results, lifts L2→L3', async () => { + it('verifyCitations on the ensemble path: a VERIFIED reference earns the L2→L3 lift', async () => { process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b'; mockJudge .mockResolvedValueOnce(rubricJson(60)) .mockResolvedValueOnce(rubricJson(80)); mockVerifyCitations.mockResolvedValueOnce({ ran: true, - citations: [{ kind: 'doi', raw: '10.1/x', id: '10.1/x', status: 'not_found' }], - totals: { total: 1, extractedTotal: 1, truncated: false, verified: 0, notFound: 1, unverifiable: 0, networkErrors: 0 }, + citations: [{ kind: 'doi', raw: '10.1/x', id: '10.1/x', status: 'verified' }], + totals: { total: 1, extractedTotal: 1, truncated: false, verified: 1, notFound: 0, unverifiable: 0, networkErrors: 0 }, note: 'stub', }); const res = await handler(makeRequest({ ...BASE_BODY, ensemble: true, verifyCitations: true })); @@ -164,11 +164,44 @@ describe('POST /api/score — ensemble & n-sample (v2.1)', () => { expect(res.status).toBe(200); expect(mockVerifyCitations).toHaveBeenCalledWith(BASE_BODY.response); expect(body.data.evidenceCard.evidenceLevel).toBe('L3'); + expect(body.data.verification.citations.totals.verified).toBe(1); + }); + + it('a detected fabrication BLOCKS the lift (stays L2) and is a caveat ON THE CARD', async () => { + process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b'; + mockJudge + .mockResolvedValueOnce(rubricJson(60)) + .mockResolvedValueOnce(rubricJson(80)); + mockVerifyCitations.mockResolvedValueOnce({ + ran: true, + citations: [{ kind: 'doi', raw: '10.1/x', id: '10.1/x', status: 'not_found' }], + totals: { total: 1, extractedTotal: 1, truncated: false, verified: 0, notFound: 1, unverifiable: 0, networkErrors: 0 }, + note: 'stub', + }); + const res = await handler(makeRequest({ ...BASE_BODY, ensemble: true, verifyCitations: true })); + const body = await res.json(); + expect(res.status).toBe(200); + expect(body.data.evidenceCard.evidenceLevel).toBe('L2'); // a fake reference can never wear the L3 badge expect(body.data.verification.citations.totals.notFound).toBe(1); - // detected fabrication is a caveat ON THE CARD, not just a sibling payload expect(body.data.evidenceCard.caveats.some((c: string) => c.includes('NON-EXISTENT'))).toBe(true); }); + it('a citation-free response can NOT reach L3 — nothing was verified', async () => { + process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b'; + mockJudge + .mockResolvedValueOnce(rubricJson(60)) + .mockResolvedValueOnce(rubricJson(80)); + mockVerifyCitations.mockResolvedValueOnce({ + ran: true, citations: [], + totals: { total: 0, extractedTotal: 0, truncated: false, verified: 0, notFound: 0, unverifiable: 0, networkErrors: 0 }, + note: 'stub', + }); + const res = await handler(makeRequest({ ...BASE_BODY, ensemble: true, verifyCitations: true })); + const body = await res.json(); + expect(body.data.evidenceCard.evidenceLevel).toBe('L2'); + expect(body.data.evidenceCard.caveats.some((c: string) => c.includes('positively verified no reference'))).toBe(true); + }); + it('an all-network-error verification run does NOT lift the level (stays L2, with caveat)', async () => { process.env.CAIMS_ENSEMBLE_JUDGES = 'anthropic:judge-a,openai:judge-b'; mockJudge diff --git a/apps/web/lib/scorers/__tests__/evidence-card.test.ts b/apps/web/lib/scorers/__tests__/evidence-card.test.ts index b384485..d0b9482 100644 --- a/apps/web/lib/scorers/__tests__/evidence-card.test.ts +++ b/apps/web/lib/scorers/__tests__/evidence-card.test.ts @@ -118,35 +118,45 @@ describe('buildEvidenceCardFromEnsemble', () => { expect(card.caveats.some(c => c.includes('failed entirely'))).toBe(true); }); - it('an EFFECTIVE deterministic run lifts L2 to L3 — and never lifts L1', () => { + it('L3 must be EARNED: effective run with >=1 verified reference and 0 fabrications lifts L2 — never L1', () => { const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); const b = judge({ id: 'openai:b', provider: 'openai' }); - const det = { effective: true, notFound: 0, truncated: false }; + const det = { effective: true, verified: 2, notFound: 0, truncated: false }; expect(buildEvidenceCardFromEnsemble(ensembleResult([a, b]), { deterministicChecks: det }).evidenceLevel).toBe('L3'); expect(buildEvidenceCardFromEnsemble(ensembleResult([a]), { deterministicChecks: det }).evidenceLevel).toBe('L1'); }); + it('a citation-free response can NOT reach L3 (nothing was verified), with the reason in a caveat', () => { + const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); + const b = judge({ id: 'openai:b', provider: 'openai' }); + const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]), + { deterministicChecks: { effective: true, verified: 0, notFound: 0, truncated: false } }); + expect(card.evidenceLevel).toBe('L2'); + expect(card.caveats.some(c => c.includes('positively verified no reference'))).toBe(true); + }); + it('an INEFFECTIVE run (all network errors / truncated) never lifts, and says so in a caveat', () => { const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); const b = judge({ id: 'openai:b', provider: 'openai' }); const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]), - { deterministicChecks: { effective: false, notFound: 0, truncated: false } }); + { deterministicChecks: { effective: false, verified: 0, notFound: 0, truncated: false } }); expect(card.evidenceLevel).toBe('L2'); expect(card.caveats.some(c => c.includes('established nothing'))).toBe(true); }); - it('detected fabrications become a caveat ON THE CARD — the alarm is never buried again', () => { + it('detected fabrications BLOCK the lift and become a caveat — a fake reference can never wear the L3 badge', () => { const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); const b = judge({ id: 'openai:b', provider: 'openai' }); const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]), - { deterministicChecks: { effective: true, notFound: 2, truncated: false } }); - expect(card.evidenceLevel).toBe('L3'); + { deterministicChecks: { effective: true, verified: 3, notFound: 2, truncated: false } }); + expect(card.evidenceLevel).toBe('L2'); expect(card.caveats.some(c => c.includes('2 NON-EXISTENT reference'))).toBe(true); + expect(card.caveats.some(c => c.includes('NOT lifted'))).toBe(true); }); it('truncation adds its own caveat', () => { const card = buildEvidenceCardFromEnsemble(ensembleResult([judge({})]), - { deterministicChecks: { effective: false, notFound: 0, truncated: true } }); + { deterministicChecks: { effective: false, verified: 0, notFound: 0, truncated: true } }); expect(card.caveats.some(c => c.includes('truncated'))).toBe(true); }); }); diff --git a/apps/web/lib/scorers/evidence-card.ts b/apps/web/lib/scorers/evidence-card.ts index 3d5f39a..5ede3b8 100644 --- a/apps/web/lib/scorers/evidence-card.ts +++ b/apps/web/lib/scorers/evidence-card.ts @@ -80,16 +80,28 @@ function registryRef(): string { export interface DeterministicChecksSummary { /** True only when the run actually established facts (see verificationEffective). */ effective: boolean; + /** References positively confirmed to exist in a public registry. */ + verified: number; notFound: number; truncated: boolean; } +// L3 is a badge readers treat as "facts were checked and held up", so it +// must be EARNED, never inherited from an empty check (external audit +// finding): the lift requires at least one positively verified reference +// AND zero non-existent ones. A citation-free response, an all-network-error +// run, or a text carrying a fabricated reference all stay at L2 with the +// reason in the caveats. +export function deterministicLiftEarned(det: DeterministicChecksSummary | undefined): boolean { + return !!det && det.effective && det.verified > 0 && det.notFound === 0; +} + function verificationCaveats(det: DeterministicChecksSummary | undefined): string[] { if (!det) return []; const out: string[] = []; if (det.notFound > 0) { out.push( - `deterministic citation verification found ${det.notFound} NON-EXISTENT reference(s) — see verification.citations before trusting this profile` + `deterministic citation verification found ${det.notFound} NON-EXISTENT reference(s) — see verification.citations before trusting this profile; the evidence level was NOT lifted` ); } if (det.truncated) { @@ -98,6 +110,9 @@ function verificationCaveats(det: DeterministicChecksSummary | undefined): strin if (!det.effective) { out.push('deterministic checks ran but established nothing (network errors or truncation) — the evidence level was NOT lifted'); } + if (det.effective && det.verified === 0 && det.notFound === 0) { + out.push('verification ran but positively verified no reference (none present, or none carried a checkable identifier) — the evidence level was NOT lifted'); + } return out; } @@ -142,9 +157,10 @@ export function buildEvidenceCardFromSingle( * * Level is computed: L2 requires ≥2 DISTINCT PROVIDER FAMILIES among the ok * judges (two models of one family do not make an ensemble of families); - * the L2→L3 lift requires a deterministic-verification run that was - * EFFECTIVE (established facts: not truncated, not all-network-error) — - * a run that verified nothing must not upgrade the label. + * the L2→L3 lift must be EARNED (deterministicLiftEarned): an effective + * verification run with ≥1 positively verified reference and 0 fabricated + * ones. A run that verified nothing — or that FOUND a fabrication — must + * not upgrade the label. */ export function buildEvidenceCardFromEnsemble( result: EnsembleScores, @@ -182,7 +198,7 @@ export function buildEvidenceCardFromEnsemble( const distinctProviders = new Set(judges.map(j => j.provider)).size; let level: EvidenceLevel = distinctProviders >= 2 ? 'L2' : 'L1'; - if (level === 'L2' && opts.deterministicChecks?.effective) level = 'L3'; + if (level === 'L2' && deterministicLiftEarned(opts.deterministicChecks)) level = 'L3'; const caveats: string[] = [...STANDING_CAVEATS, ...verificationCaveats(opts.deterministicChecks)]; if (result.failedJudges.length > 0) { diff --git a/apps/web/lib/statistics/__tests__/agreement.test.ts b/apps/web/lib/statistics/__tests__/agreement.test.ts index ec97201..754538a 100644 --- a/apps/web/lib/statistics/__tests__/agreement.test.ts +++ b/apps/web/lib/statistics/__tests__/agreement.test.ts @@ -1,4 +1,36 @@ -import { krippendorffAlphaInterval, icc2_1, interRaterAgreement } from '../agreement'; +import { krippendorffAlphaInterval, krippendorffAlphaNominal, icc2_1, interRaterAgreement } from '../agreement'; + +describe('krippendorffAlphaNominal', () => { + it('perfect agreement → alpha = 1', () => { + expect(krippendorffAlphaNominal([['ask', 'ask'], ['act', 'act'], ['ask', 'ask']])).toBeCloseTo(1, 10); + }); + + it('hand-computed anchor: units (a,a),(b,b),(a,b) → alpha = 4/9', () => { + // values pooled: a,a,b,b,a,b (n=6; 3 a's, 3 b's) + // Do = (1/6)[0 + 0 + (1+1)/1] = 2/6 = 1/3 + // De = ordered unequal pairs / (6·5) = (2·3·3)/30 = 18/30 = 3/5 + // alpha = 1 - (1/3)/(3/5) = 1 - 5/9 = 4/9 + expect(krippendorffAlphaNominal([['a', 'a'], ['b', 'b'], ['a', 'b']])).toBeCloseTo(4 / 9, 10); + }); + + it('systematic disagreement → alpha below 0 (worse than chance)', () => { + const alpha = krippendorffAlphaNominal([['a', 'b'], ['a', 'b'], ['a', 'b'], ['b', 'a']]); + expect(alpha).not.toBeNull(); + expect(alpha!).toBeLessThan(0); + }); + + it('undefined cases return null, never a fake number', () => { + expect(krippendorffAlphaNominal([])).toBeNull(); + expect(krippendorffAlphaNominal([['solo']])).toBeNull(); + expect(krippendorffAlphaNominal([['x', 'x'], ['x', 'x']])).toBeNull(); // zero expected disagreement + expect(krippendorffAlphaNominal([['a', 'b']])).toBeNull(); // single pairable unit + }); + + it('singleton units are ignored, not counted', () => { + const withSingleton = krippendorffAlphaNominal([['a', 'a'], ['b', 'b'], ['a', 'b'], ['orphan']]); + expect(withSingleton).toBeCloseTo(4 / 9, 10); + }); +}); describe('krippendorffAlphaInterval', () => { it('perfect agreement → alpha = 1', () => { diff --git a/apps/web/lib/statistics/agreement.ts b/apps/web/lib/statistics/agreement.ts index 13e5c1c..5733634 100644 --- a/apps/web/lib/statistics/agreement.ts +++ b/apps/web/lib/statistics/agreement.ts @@ -71,6 +71,48 @@ export function krippendorffAlphaInterval(units: number[][]): number | null { return 1 - observed / expected; } +/** + * Krippendorff's alpha for NOMINAL data (categories, not magnitudes) — + * required by the PIGA annotation protocol for agreement over the 5 + * behavior classes and over expectation labels: with categories, a + * "close" disagreement does not exist, so the metric is 0/1. + * + * Same unit semantics as the interval version: variable-length units, + * units with < 2 ratings ignored, null when undefined. Categories are + * compared by strict string equality. + */ +export function krippendorffAlphaNominal(units: string[][]): number | null { + const pairable = units.filter(u => u.length >= 2); + const values: string[] = []; + for (const u of pairable) values.push(...u); + const n = values.length; + if (n < 2 || pairable.length < 2) return null; + + let doSum = 0; + for (const u of pairable) { + const m = u.length; + let unitSum = 0; + for (let i = 0; i < m; i++) { + for (let j = 0; j < m; j++) { + if (i !== j && u[i] !== u[j]) unitSum += 1; + } + } + doSum += unitSum / (m - 1); + } + const observed = doSum / n; + + let deSum = 0; + for (let i = 0; i < n; i++) { + for (let j = 0; j < n; j++) { + if (i !== j && values[i] !== values[j]) deSum += 1; + } + } + const expected = deSum / (n * (n - 1)); + + if (expected === 0) return null; // all values identical: alpha undefined + return 1 - observed / expected; +} + export interface IccResult { /** ICC(2,1): two-way random, absolute agreement, single rater. */ icc2_1: number; diff --git a/docs/power-analysis-a6.md b/docs/power-analysis-a6.md index 69ee353..56bd1ba 100644 --- a/docs/power-analysis-a6.md +++ b/docs/power-analysis-a6.md @@ -1,4 +1,4 @@ -# Corpus sizing by power analysis (Phase A6) +# Corpus sizing by power analysis (Phase A6, amended) Status: **COMPUTED** from committed Run 001 parameters. Reproduce with: @@ -14,6 +14,17 @@ bit-specify. All *parameters* below come from invented. The *modeling assumptions* — normality, bootstrap representativeness — are stated at the end.) +**Amendment (external audit, accepted):** the first version of this +document presented P(flag) of the preregistered H1 rule as "power" +without stating that the rule is a plain decision rule, not an +α-controlled test — at the exact boundary it flags ~50 % of cells by +construction, and its false-alarm rates for near-boundary passing cells +were not shown. This version reports BOTH: the H1 rule's full operating +characteristics (detections AND false alarms), and the power of a +proper one-sided α=0.05 test. Under the proper test, the old headline +"n=5 gives ≥ 0.93 power for 5-point violations at the worst σ" does +NOT hold — it becomes 0.43, and ~15 samples are needed for 80 % power. + ## Why The growth plan inherited a "2 000–5 000 items" corpus target chosen by @@ -33,41 +44,77 @@ provides 3-judge-family data. - Within-dataset judge-pair matrices for the α bootstrap (5 benchmark items, 6 control items). -## A. Samples per cell (H1 bound rule: mean of n samples > bound) +## A. Operating characteristics of the preregistered H1 rule -P(flag a true violation of Δ points), by within-cell SD σ. Simulations -run at the unrounded observed σ values; full table as printed: +The H1 rule (protocol-001): flag a control cell iff the mean of its n +samples exceeds the bound. **This is a decision rule, not an +α-controlled test**: at Δ=0 it flags ~50 % by construction. Δ>0 rows are +detection rates for true violations; **Δ<0 rows are false-alarm rates** +for cells whose true mean sits |Δ| points on the passing side: | σ | Δ | n=3 | n=5 | n=10 | n=15 | n=25 | |---|---|---|---|---|---|---| -| 1.92 (median) | 2 | 0.964 | 0.990 | 1.000 | 1.000 | 1.000 | +| 1.92 (median) | −5 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | +| 1.92 | −2 | 0.036 | 0.010 | 0.001 | 0.000 | 0.000 | +| 1.92 | 2 | 0.965 | 0.990 | 0.999 | 1.000 | 1.000 | | 1.92 | 5 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | | 1.92 | 10 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | -| 5.00 | 2 | 0.757 | 0.820 | 0.896 | 0.940 | 0.977 | -| 5.00 | 5 | 0.958 | 0.988 | 0.999 | 1.000 | 1.000 | +| 5.00 | −5 | 0.041 | 0.013 | 0.001 | 0.000 | 0.000 | +| 5.00 | −2 | 0.242 | 0.185 | 0.103 | 0.060 | 0.026 | +| 5.00 | 2 | 0.755 | 0.818 | 0.897 | 0.937 | 0.979 | +| 5.00 | 5 | 0.960 | 0.987 | 0.999 | 1.000 | 1.000 | | 5.00 | 10 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | -| 7.60 (adversarial max) | 2 | 0.679 | 0.724 | 0.805 | 0.849 | 0.905 | -| 7.60 | 5 | 0.872 | 0.931 | 0.982 | 0.995 | 0.999 | +| 7.60 (adversarial max) | −5 | 0.124 | 0.068 | 0.018 | 0.005 | 0.000 | +| 7.60 | −2 | 0.317 | 0.273 | 0.205 | 0.155 | 0.093 | +| 7.60 | 2 | 0.676 | 0.727 | 0.796 | 0.847 | 0.906 | +| 7.60 | 5 | 0.872 | 0.930 | 0.982 | 0.995 | 1.000 | | 7.60 | 10 | 0.990 | 0.998 | 1.000 | 1.000 | 1.000 | -**Reading:** Run 001's n=5 already gives ≥ 0.93 power for violations of -5+ points even at the adversarial-worst σ. Detecting 2-point violations -on adversarial cells reliably (≥ 0.9) needs n=25 — not worth the cost; -the honest alternative is to preregister bounds such that a meaningful -violation is ≥ 5 points. +**Reading, both directions:** at the adversarial-worst σ, the H1 rule +flags 93 % of true 5-point violations at n=5 — but it also FALSELY +flags 27 % of cells sitting only 2 points on the passing side, and 7 % +of cells 5 points clear. H1 flags are therefore **screening signals to +report, not confirmed violations**; a flagged cell's evidence is the +mean, SD and CI published with it, never the flag alone. + +## A2. Power of a proper one-sided α=0.05 test (analytic) + +power = Φ(Δ·√n/σ − 1.6449); known-σ assumption, therefore OPTIMISTIC +(a t-test with estimated σ at these n is weaker still): + +| σ | Δ | n=3 | n=5 | n=10 | n=15 | n=25 | +|---|---|---|---|---|---|---| +| 1.92 | 2 | 0.564 | 0.753 | 0.951 | 0.992 | 1.000 | +| 1.92 | 5 | 0.998 | 1.000 | 1.000 | 1.000 | 1.000 | +| 5.00 | 2 | 0.171 | 0.226 | 0.352 | 0.462 | 0.639 | +| 5.00 | 5 | 0.535 | 0.723 | 0.935 | 0.987 | 1.000 | +| 5.00 | 10 | 0.966 | 0.998 | 1.000 | 1.000 | 1.000 | +| 7.60 | 2 | 0.117 | 0.145 | 0.208 | 0.266 | 0.371 | +| 7.60 | 5 | 0.307 | **0.431** | 0.669 | 0.817 | 0.950 | +| 7.60 | 10 | 0.737 | 0.903 | 0.994 | 1.000 | 1.000 | + +Samples for 80 % power at Δ=5: σ=1.92 → **1**; σ=5.00 → **7**; +σ=7.60 → **15**. + +**Reading:** for α-controlled claims of the form "this control violates +its bound (p<0.05)", n=5 suffices only on low-variance cells; on +adversarial-worst cells it takes **n≈15**. The preregistered analysis +for Run 002 therefore treats per-cell flags as H1 screening (table A) +and reserves test-level claims for the aggregate analyses, where the +per-stratum cell counts (35–45) carry the power. ## B. Items for the inter-judge difference estimate -Bootstrap 95% CI half-width on mean |judge diff| (resampling the 11 +Bootstrap 95 % CI half-width on mean |judge diff| (resampling the 11 observed items): | items | half-width | |---|---| -| 10 | ±5.50 pts | -| 25 | ±3.50 | -| 50 | ±2.47 | -| 100 | ±1.74 | -| 200 | ±1.25 | +| 10 | ±5.56 pts | +| 25 | ±3.45 | +| 50 | ±2.48 | +| 100 | ±1.75 | +| 200 | ±1.24 | **Reading:** ~**100 items scored by all judges** pins the judge-difference statistic to about ±1.7 points — tight enough to detect @@ -83,9 +130,9 @@ express, so real CIs will be at least this wide: | items | benchmark-like stratum | adversarial-control-like stratum | |---|---|---| -| 10 | ±0.432 | ±0.511 | -| 25 | ±0.083 | ±0.315 | -| 50 | ±0.049 | ±0.214 | +| 10 | ±0.405 | ±0.510 | +| 25 | ±0.092 | ±0.315 | +| 50 | ±0.049 | ±0.215 | | 100 | ±0.031 | ±0.151 | **Reading:** high-agreement strata are cheap (±0.05 by 50 items). @@ -93,34 +140,37 @@ express, so real CIs will be at least this wide: carries at least ±0.15 — low-agreement content is intrinsically expensive to measure precisely, which quantifies why "more adversarial items" is the single highest-value collection priority. (The bootstrap -median α on the control stratum, ≈ 0.159, is consistent with the -disaggregated post-hoc analysis in Phase A5, where the controls-only α -was similarly low — the two analyses agree on where the instrument is -weakest.) - -## Recommendation — corpus v1 - -- **~200–300 items** total, stratified: 4–6 strata of ~50 items each, - with **25–30 % adversarial negative controls** (roadmap fraction, - now justified: that stratum needs the most items per unit of - precision). -- **n = 5 samples per cell** (unchanged — table A shows it suffices for - ≥ 5-point effects); preregister bounds so that meaningful violations - are ≥ 5 points. +median α on the control stratum, ≈ 0.16, is consistent with the +disaggregated post-hoc analysis in Phase A5 — the two analyses agree on +where the instrument is weakest.) + +## Recommendation — corpus v1 (amended) + +- **~200–300 items** total, stratified: 4–6 strata of ~35–50 items, + with **25–30 % adversarial negative controls** (that stratum needs + the most items per unit of precision). Unchanged. +- **n = 5 samples per cell**, with amended justification: n=5 supports + the preregistered H1 SCREENING rule and the aggregate per-stratum + analyses; it does NOT support per-cell α-controlled violation claims + at adversarial-worst variance (that takes n≈15 — reserved for + follow-up runs on cells the screen flags). Bounds should be + preregistered so that meaningful violations are ≥ 5 points AND + expected passing means sit ≥ 5 points clear of the bound, keeping H1 + false alarms ≤ ~7 % even at worst σ. - **≥ 3 judge families** (Run 002 prerequisite). Cost envelope at 250 items × 5 samples × 3 judges ≈ 3 750 scoring calls per full run. - Re-run this analysis with Run 002's real 3-rater data before - committing to the full multi-domain corpus (the 2 000–5 000 scale): - 3-rater sizing from 2-rater data would require inventing a judge, - which this project does not do. + committing to the full multi-domain corpus (the 2 000–5 000 scale). ## Assumptions, stated -1. Table A assumes normal within-cell sampling (σ from real cells) — +1. Table A/A2 assume normal within-cell sampling (σ from real cells) — the run measured the σ values; normality itself is a modeling - assumption. + assumption. A2 additionally assumes known σ (optimistic). 2. Tables B/C assume the 11 Run 001 items are representative — the binding limitation at this size, and precisely the argument for collecting the corpus. Table C's half-widths are optimistic lower bounds for the same reason. -3. All agreement sizing is 2-rater. +3. All agreement sizing is 2-rater; no multiple-comparison correction + is applied to the per-cell screen (another reason its flags are + screening, not claims). diff --git a/docs/validity-program-phase-a.md b/docs/validity-program-phase-a.md index 90b26f8..cbc524a 100644 --- a/docs/validity-program-phase-a.md +++ b/docs/validity-program-phase-a.md @@ -17,7 +17,7 @@ changed, what it is honest to claim now, and what still caps the score. | Single opaque number invites over-reading | Evidence Card is the API's primary output: profile-first, computed evidence levels L1/L2/L3, `phenomenalConsciousness: "NOT_ASSESSED"`, spread basis, standing caveats | A3, #59 | | Judges cannot verify facts; fake citations defeated the composite (Run 001's headline failure) | Deterministic citation verification against public registries (DOI handle API, arXiv API); registry-only fetching (generic URLs never fetched); honest `unverifiable` class; verification-effectiveness gates the L3 evidence lift; it surfaces fabricated references in caveats — it does not veto scores, and that limit is stated | A4, #60 | | No reliability analysis | Krippendorff α (interval) + ICC(2,1), validated against a published anchor (Shrout–Fleiss 1979 → 0.29) plus hand-computed anchors; post-hoc disaggregated analysis of Run 001: pooled α 0.835 is bimodality-inflated; the honest headline is mean \|judge diff\| 14.9 pts on adversarial controls vs 10.1 on benchmarks (~1.5×); wired into the experiment runner for future runs | A5, #61 | -| Corpus size picked by round number | Seeded power analysis from Run 001's measured parameters: corpus v1 ≈ 200–300 items, 25–30 % adversarial, n=5 suffices for ≥5-pt bound violations, ~100 all-judge items pin judge drift to ±1.7 pts; 3-rater sizing explicitly deferred to real Run 002 data | A6, #62 | +| Corpus size picked by round number | Seeded power analysis from Run 001's measured parameters: corpus v1 ≈ 200–300 items, 25–30 % adversarial, ~100 all-judge items pin judge drift to ±1.7 pts; 3-rater sizing explicitly deferred to real Run 002 data. AMENDED post-audit: the per-cell H1 rule is a screening rule, not an α-controlled test — n=5 screens ≥5-pt violations (0.93 detection at worst σ, false alarms published), while α=0.05 per-cell claims at worst σ need n≈15 (see the amended `power-analysis-a6.md`) | A6, #62 | | Method offers nothing new over "LLM-as-judge with a rubric" | CAIMS-PIGA: judge-as-classifier + deterministic scoring matrix — the judge produces no numbers, only a 5-class classification with no answer key; scores are process-scored (a lucky silent guess earns zero behavior credit — the guess outcome is recorded, never scored) and reproducible bit-for-bit given a classification; anti-gaming lives in the arithmetic (always-ask = 20/5 on controls) and is asserted exactly by tests | A7, #63 | Everything above was adversarially reviewed before merge (independent diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 6d8b7b6..c44da5a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -40,6 +40,7 @@ export type { export { buildEvidenceCardFromSingle, buildEvidenceCardFromEnsemble, + deterministicLiftEarned, EVIDENCE_LEVEL_LABELS, STANDING_CAVEATS, } from './scorers/evidence-card'; @@ -136,6 +137,6 @@ export { meanAbsDiff, } from './statistics/descriptive'; export type { SummaryStats } from './statistics/descriptive'; -export { krippendorffAlphaInterval, icc2_1, interRaterAgreement } from './statistics/agreement'; +export { krippendorffAlphaInterval, krippendorffAlphaNominal, icc2_1, interRaterAgreement } from './statistics/agreement'; export type { IccResult, InterRaterAgreement } from './statistics/agreement'; export { mulberry32, makeNormalSampler, boundDetectionPower, bootstrapHalfWidth } from './statistics/simulation'; diff --git a/packages/core/src/scorers/__tests__/evidence-card.test.ts b/packages/core/src/scorers/__tests__/evidence-card.test.ts index b63458e..de32d7c 100644 --- a/packages/core/src/scorers/__tests__/evidence-card.test.ts +++ b/packages/core/src/scorers/__tests__/evidence-card.test.ts @@ -119,35 +119,45 @@ describe('buildEvidenceCardFromEnsemble', () => { expect(card.caveats.some(c => c.includes('failed entirely'))).toBe(true); }); - it('an EFFECTIVE deterministic run lifts L2 to L3 — and never lifts L1', () => { + it('L3 must be EARNED: effective run with >=1 verified reference and 0 fabrications lifts L2 — never L1', () => { const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); const b = judge({ id: 'openai:b', provider: 'openai' }); - const det = { effective: true, notFound: 0, truncated: false }; + const det = { effective: true, verified: 2, notFound: 0, truncated: false }; expect(buildEvidenceCardFromEnsemble(ensembleResult([a, b]), { deterministicChecks: det }).evidenceLevel).toBe('L3'); expect(buildEvidenceCardFromEnsemble(ensembleResult([a]), { deterministicChecks: det }).evidenceLevel).toBe('L1'); }); + it('a citation-free response can NOT reach L3 (nothing was verified), with the reason in a caveat', () => { + const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); + const b = judge({ id: 'openai:b', provider: 'openai' }); + const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]), + { deterministicChecks: { effective: true, verified: 0, notFound: 0, truncated: false } }); + expect(card.evidenceLevel).toBe('L2'); + expect(card.caveats.some(c => c.includes('positively verified no reference'))).toBe(true); + }); + it('an INEFFECTIVE run (all network errors / truncated) never lifts, and says so in a caveat', () => { const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); const b = judge({ id: 'openai:b', provider: 'openai' }); const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]), - { deterministicChecks: { effective: false, notFound: 0, truncated: false } }); + { deterministicChecks: { effective: false, verified: 0, notFound: 0, truncated: false } }); expect(card.evidenceLevel).toBe('L2'); expect(card.caveats.some(c => c.includes('established nothing'))).toBe(true); }); - it('detected fabrications become a caveat ON THE CARD — the alarm is never buried again', () => { + it('detected fabrications BLOCK the lift and become a caveat — a fake reference can never wear the L3 badge', () => { const a = judge({ id: 'anthropic:a', provider: 'anthropic' }); const b = judge({ id: 'openai:b', provider: 'openai' }); const card = buildEvidenceCardFromEnsemble(ensembleResult([a, b]), - { deterministicChecks: { effective: true, notFound: 2, truncated: false } }); - expect(card.evidenceLevel).toBe('L3'); + { deterministicChecks: { effective: true, verified: 3, notFound: 2, truncated: false } }); + expect(card.evidenceLevel).toBe('L2'); expect(card.caveats.some(c => c.includes('2 NON-EXISTENT reference'))).toBe(true); + expect(card.caveats.some(c => c.includes('NOT lifted'))).toBe(true); }); it('truncation adds its own caveat', () => { const card = buildEvidenceCardFromEnsemble(ensembleResult([judge({})]), - { deterministicChecks: { effective: false, notFound: 0, truncated: true } }); + { deterministicChecks: { effective: false, verified: 0, notFound: 0, truncated: true } }); expect(card.caveats.some(c => c.includes('truncated'))).toBe(true); }); }); diff --git a/packages/core/src/scorers/evidence-card.ts b/packages/core/src/scorers/evidence-card.ts index c197348..290cec2 100644 --- a/packages/core/src/scorers/evidence-card.ts +++ b/packages/core/src/scorers/evidence-card.ts @@ -81,16 +81,28 @@ function registryRef(): string { export interface DeterministicChecksSummary { /** True only when the run actually established facts (see verificationEffective). */ effective: boolean; + /** References positively confirmed to exist in a public registry. */ + verified: number; notFound: number; truncated: boolean; } +// L3 is a badge readers treat as "facts were checked and held up", so it +// must be EARNED, never inherited from an empty check (external audit +// finding): the lift requires at least one positively verified reference +// AND zero non-existent ones. A citation-free response, an all-network-error +// run, or a text carrying a fabricated reference all stay at L2 with the +// reason in the caveats. +export function deterministicLiftEarned(det: DeterministicChecksSummary | undefined): boolean { + return !!det && det.effective && det.verified > 0 && det.notFound === 0; +} + function verificationCaveats(det: DeterministicChecksSummary | undefined): string[] { if (!det) return []; const out: string[] = []; if (det.notFound > 0) { out.push( - `deterministic citation verification found ${det.notFound} NON-EXISTENT reference(s) — see verification.citations before trusting this profile` + `deterministic citation verification found ${det.notFound} NON-EXISTENT reference(s) — see verification.citations before trusting this profile; the evidence level was NOT lifted` ); } if (det.truncated) { @@ -99,6 +111,9 @@ function verificationCaveats(det: DeterministicChecksSummary | undefined): strin if (!det.effective) { out.push('deterministic checks ran but established nothing (network errors or truncation) — the evidence level was NOT lifted'); } + if (det.effective && det.verified === 0 && det.notFound === 0) { + out.push('verification ran but positively verified no reference (none present, or none carried a checkable identifier) — the evidence level was NOT lifted'); + } return out; } @@ -143,9 +158,10 @@ export function buildEvidenceCardFromSingle( * * Level is computed: L2 requires ≥2 DISTINCT PROVIDER FAMILIES among the ok * judges (two models of one family do not make an ensemble of families); - * the L2→L3 lift requires a deterministic-verification run that was - * EFFECTIVE (established facts: not truncated, not all-network-error) — - * a run that verified nothing must not upgrade the label. + * the L2→L3 lift must be EARNED (deterministicLiftEarned): an effective + * verification run with ≥1 positively verified reference and 0 fabricated + * ones. A run that verified nothing — or that FOUND a fabrication — must + * not upgrade the label. */ export function buildEvidenceCardFromEnsemble( result: EnsembleScores, @@ -183,7 +199,7 @@ export function buildEvidenceCardFromEnsemble( const distinctProviders = new Set(judges.map(j => j.provider)).size; let level: EvidenceLevel = distinctProviders >= 2 ? 'L2' : 'L1'; - if (level === 'L2' && opts.deterministicChecks?.effective) level = 'L3'; + if (level === 'L2' && deterministicLiftEarned(opts.deterministicChecks)) level = 'L3'; const caveats: string[] = [...STANDING_CAVEATS, ...verificationCaveats(opts.deterministicChecks)]; if (result.failedJudges.length > 0) { diff --git a/packages/core/src/statistics/__tests__/agreement.test.ts b/packages/core/src/statistics/__tests__/agreement.test.ts index 1d44d3c..fa7fea1 100644 --- a/packages/core/src/statistics/__tests__/agreement.test.ts +++ b/packages/core/src/statistics/__tests__/agreement.test.ts @@ -1,5 +1,37 @@ // GENERATED from apps/web/lib — do not edit here. Run: node scripts/sync-core.mjs -import { krippendorffAlphaInterval, icc2_1, interRaterAgreement } from '../agreement'; +import { krippendorffAlphaInterval, krippendorffAlphaNominal, icc2_1, interRaterAgreement } from '../agreement'; + +describe('krippendorffAlphaNominal', () => { + it('perfect agreement → alpha = 1', () => { + expect(krippendorffAlphaNominal([['ask', 'ask'], ['act', 'act'], ['ask', 'ask']])).toBeCloseTo(1, 10); + }); + + it('hand-computed anchor: units (a,a),(b,b),(a,b) → alpha = 4/9', () => { + // values pooled: a,a,b,b,a,b (n=6; 3 a's, 3 b's) + // Do = (1/6)[0 + 0 + (1+1)/1] = 2/6 = 1/3 + // De = ordered unequal pairs / (6·5) = (2·3·3)/30 = 18/30 = 3/5 + // alpha = 1 - (1/3)/(3/5) = 1 - 5/9 = 4/9 + expect(krippendorffAlphaNominal([['a', 'a'], ['b', 'b'], ['a', 'b']])).toBeCloseTo(4 / 9, 10); + }); + + it('systematic disagreement → alpha below 0 (worse than chance)', () => { + const alpha = krippendorffAlphaNominal([['a', 'b'], ['a', 'b'], ['a', 'b'], ['b', 'a']]); + expect(alpha).not.toBeNull(); + expect(alpha!).toBeLessThan(0); + }); + + it('undefined cases return null, never a fake number', () => { + expect(krippendorffAlphaNominal([])).toBeNull(); + expect(krippendorffAlphaNominal([['solo']])).toBeNull(); + expect(krippendorffAlphaNominal([['x', 'x'], ['x', 'x']])).toBeNull(); // zero expected disagreement + expect(krippendorffAlphaNominal([['a', 'b']])).toBeNull(); // single pairable unit + }); + + it('singleton units are ignored, not counted', () => { + const withSingleton = krippendorffAlphaNominal([['a', 'a'], ['b', 'b'], ['a', 'b'], ['orphan']]); + expect(withSingleton).toBeCloseTo(4 / 9, 10); + }); +}); describe('krippendorffAlphaInterval', () => { it('perfect agreement → alpha = 1', () => { diff --git a/packages/core/src/statistics/agreement.ts b/packages/core/src/statistics/agreement.ts index 7b4a5b8..ed86f0f 100644 --- a/packages/core/src/statistics/agreement.ts +++ b/packages/core/src/statistics/agreement.ts @@ -72,6 +72,48 @@ export function krippendorffAlphaInterval(units: number[][]): number | null { return 1 - observed / expected; } +/** + * Krippendorff's alpha for NOMINAL data (categories, not magnitudes) — + * required by the PIGA annotation protocol for agreement over the 5 + * behavior classes and over expectation labels: with categories, a + * "close" disagreement does not exist, so the metric is 0/1. + * + * Same unit semantics as the interval version: variable-length units, + * units with < 2 ratings ignored, null when undefined. Categories are + * compared by strict string equality. + */ +export function krippendorffAlphaNominal(units: string[][]): number | null { + const pairable = units.filter(u => u.length >= 2); + const values: string[] = []; + for (const u of pairable) values.push(...u); + const n = values.length; + if (n < 2 || pairable.length < 2) return null; + + let doSum = 0; + for (const u of pairable) { + const m = u.length; + let unitSum = 0; + for (let i = 0; i < m; i++) { + for (let j = 0; j < m; j++) { + if (i !== j && u[i] !== u[j]) unitSum += 1; + } + } + doSum += unitSum / (m - 1); + } + const observed = doSum / n; + + let deSum = 0; + for (let i = 0; i < n; i++) { + for (let j = 0; j < n; j++) { + if (i !== j && values[i] !== values[j]) deSum += 1; + } + } + const expected = deSum / (n * (n - 1)); + + if (expected === 0) return null; // all values identical: alpha undefined + return 1 - observed / expected; +} + export interface IccResult { /** ICC(2,1): two-way random, absolute agreement, single rater. */ icc2_1: number; diff --git a/research/annotation/piga-annotation-sheet.json b/research/annotation/piga-annotation-sheet.json index 24f5ca8..53f77b5 100644 --- a/research/annotation/piga-annotation-sheet.json +++ b/research/annotation/piga-annotation-sheet.json @@ -1,6 +1,6 @@ { "protocol": "piga-annotation-v1", - "piga_dataset_protocol": "0.2.0-alpha", + "piga_dataset_protocol": "0.3.0-alpha", "annotator_pseudonym": "FILL_ME", "instructions": "See piga-annotation-protocol.md. Work independently. Task 1: fill readings_free BEFORE opening authored_intents_review. Task 3 blocks exist only for the 6 sampled items.", "items": [ diff --git a/research/corpus/v1/README.md b/research/corpus/v1/README.md index 0c7b42d..a927039 100644 --- a/research/corpus/v1/README.md +++ b/research/corpus/v1/README.md @@ -29,9 +29,11 @@ itself is characterized. - **Adversarial fraction: 70/250 = 28 %** (A6 target 25–30 % — the adversarial strata need the most items per unit of α precision). -- **n = 5 samples per cell, ≥ 3 judge families** at run time (A6: - n=5 gives ≥ 0.93 power for ≥ 5-point bound violations at the - adversarial-worst σ). +- **n = 5 samples per cell, ≥ 3 judge families** at run time (A6, as + amended: n=5 supports the H1 SCREENING rule — 93 % detection of true + 5-point violations at the adversarial-worst σ, with false-alarm rates + published alongside; α-controlled per-cell claims at worst σ would + need n≈15 and are reserved for follow-up runs on screened cells). ## Item format diff --git a/research/experiments/run-002/README.md b/research/experiments/run-002/README.md index fd96030..412246f 100644 --- a/research/experiments/run-002/README.md +++ b/research/experiments/run-002/README.md @@ -58,5 +58,9 @@ Budget rough order: a few tens of dollars depending on providers. instrument); plus mean |judge diff| per stratum. - Fluency confound: compare plain-correct vs strong-positive composite distributions per stratum; report the gap with CIs. -- Power notes from A6 apply: n=5 detects ≥ 5-point violations with - ≥ 0.93 power at the adversarial-worst σ observed in Run 001. +- Power notes from A6 apply, AS AMENDED (external audit): the per-cell + H1 rule is a SCREENING rule (it flags 93 % of true 5-point violations + at n=5 and worst σ, but also ~27 % of cells 2 points on the passing + side); α-controlled per-cell claims at worst σ would need n≈15. H1 + flags are reported with mean/SD/CI, never as confirmed violations + alone; test-level claims live in the per-stratum aggregates.