diff --git a/.gitignore b/.gitignore index 797217f..0fd8c7a 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,7 @@ next-env.d.ts .mcp.json # DB Backups -db-backups \ No newline at end of file +db-backups + +# Eval run artifacts +/evals/results \ No newline at end of file diff --git a/evals/golden.ts b/evals/golden.ts new file mode 100644 index 0000000..29a2b08 --- /dev/null +++ b/evals/golden.ts @@ -0,0 +1,52 @@ +/** + * Golden dataset for bill-analysis evals. + * + * Each entry pins human-agreed expectations for a fixture bill. Reuses the XML + * fixtures already in tests/fixtures/, so adding a bill is just: drop its XML in + * tests/fixtures/, add an entry here. `expectedJudgment`/`expectedSocialIssue` + * are best-effort ground truth — adjust them as the team's view settles. + */ +export interface GoldenBill { + /** Fixture file basename in tests/fixtures (without .xml). */ + fixture: string; + billId: string; + /** Expected final judgment, or "any" when reasonable people could differ. */ + expectedJudgment: "yes" | "no" | "abstain" | "any"; + expectedSocialIssue: "yes" | "no"; + /** Lowercased substrings the summary must contain. */ + mustMention: string[]; +} + +export const GOLDEN_BILLS: GoldenBill[] = [ + { + fixture: "s-230", + billId: "S-230", + // Creates a new national-strategy reporting obligation (adds process). + expectedJudgment: "any", + expectedSocialIssue: "no", + mustMention: ["soil"], + }, + { + fixture: "s-231", + billId: "S-231", + expectedJudgment: "any", + expectedSocialIssue: "no", + mustMention: ["corporation"], + }, + { + fixture: "c-208", + billId: "C-208", + // Eases intergenerational small-business/farm transfers via the tax code. + expectedJudgment: "yes", + expectedSocialIssue: "no", + mustMention: ["tax"], + }, + { + fixture: "c-206", + billId: "C-206", + // Carbon-pricing exemption for farmers — reduces a cost on producers. + expectedJudgment: "any", + expectedSocialIssue: "no", + mustMention: ["carbon"], + }, +]; diff --git a/package.json b/package.json index bc1a69d..50dc201 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "test:ui": "vitest --ui", "test:coverage": "vitest run --coverage", "prompt": "tsx src/scripts/copy-bill-prompt.ts", + "eval": "tsx src/scripts/run-evals.ts", "prepare": "husky" }, "dependencies": { @@ -72,7 +73,8 @@ "remark-gfm": "^4.0.1", "sonner": "^2.0.7", "tailwind-merge": "^3.3.1", - "vaul": "^1.1.2" + "vaul": "^1.1.2", + "zod": "^3.23.8" }, "devDependencies": { "@biomejs/biome": "2.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f314661..2305a41 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -150,7 +150,7 @@ importers: version: 0.4.6(react-dom@19.1.1(react@19.1.1))(react@19.1.1) openai: specifier: ^5.15.0 - version: 5.20.2(ws@8.18.3) + version: 5.20.2(ws@8.18.3)(zod@3.25.76) react: specifier: 19.1.1 version: 19.1.1 @@ -184,6 +184,9 @@ importers: vaul: specifier: ^1.1.2 version: 1.1.2(@types/react-dom@19.1.9(@types/react@19.1.13))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + zod: + specifier: ^3.23.8 + version: 3.25.76 devDependencies: '@biomejs/biome': specifier: 2.2.0 @@ -9066,6 +9069,9 @@ packages: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -17054,9 +17060,10 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 - openai@5.20.2(ws@8.18.3): + openai@5.20.2(ws@8.18.3)(zod@3.25.76): optionalDependencies: ws: 8.18.3 + zod: 3.25.76 openid-client@5.7.1: dependencies: @@ -20102,4 +20109,6 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + zod@3.25.76: {} + zwitch@2.0.4: {} diff --git a/src/app/[id]/edit/page.tsx b/src/app/[id]/edit/page.tsx index 8e9f4b3..bd2e505 100644 --- a/src/app/[id]/edit/page.tsx +++ b/src/app/[id]/edit/page.tsx @@ -100,20 +100,6 @@ export default async function EditBillPage({ params }: Params) { className="w-full min-h-32 border rounded p-2" /> -
- -