diff --git a/.gitignore b/.gitignore
index 797217f..0fd8c7a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,4 +47,7 @@ next-env.d.ts
.mcp.json
# DB Backups
-db-backups
\ No newline at end of file
+db-backups
+
+# Eval run artifacts
+/evals/results
\ No newline at end of file
diff --git a/evals/golden.ts b/evals/golden.ts
new file mode 100644
index 0000000..29a2b08
--- /dev/null
+++ b/evals/golden.ts
@@ -0,0 +1,52 @@
+/**
+ * Golden dataset for bill-analysis evals.
+ *
+ * Each entry pins human-agreed expectations for a fixture bill. Reuses the XML
+ * fixtures already in tests/fixtures/, so adding a bill is just: drop its XML in
+ * tests/fixtures/, add an entry here. `expectedJudgment`/`expectedSocialIssue`
+ * are best-effort ground truth — adjust them as the team's view settles.
+ */
+export interface GoldenBill {
+ /** Fixture file basename in tests/fixtures (without .xml). */
+ fixture: string;
+ billId: string;
+ /** Expected final judgment, or "any" when reasonable people could differ. */
+ expectedJudgment: "yes" | "no" | "abstain" | "any";
+ expectedSocialIssue: "yes" | "no";
+ /** Lowercased substrings the summary must contain. */
+ mustMention: string[];
+}
+
+export const GOLDEN_BILLS: GoldenBill[] = [
+ {
+ fixture: "s-230",
+ billId: "S-230",
+ // Creates a new national-strategy reporting obligation (adds process).
+ expectedJudgment: "any",
+ expectedSocialIssue: "no",
+ mustMention: ["soil"],
+ },
+ {
+ fixture: "s-231",
+ billId: "S-231",
+ expectedJudgment: "any",
+ expectedSocialIssue: "no",
+ mustMention: ["corporation"],
+ },
+ {
+ fixture: "c-208",
+ billId: "C-208",
+ // Eases intergenerational small-business/farm transfers via the tax code.
+ expectedJudgment: "yes",
+ expectedSocialIssue: "no",
+ mustMention: ["tax"],
+ },
+ {
+ fixture: "c-206",
+ billId: "C-206",
+ // Carbon-pricing exemption for farmers — reduces a cost on producers.
+ expectedJudgment: "any",
+ expectedSocialIssue: "no",
+ mustMention: ["carbon"],
+ },
+];
diff --git a/package.json b/package.json
index bc1a69d..50dc201 100644
--- a/package.json
+++ b/package.json
@@ -15,6 +15,7 @@
"test:ui": "vitest --ui",
"test:coverage": "vitest run --coverage",
"prompt": "tsx src/scripts/copy-bill-prompt.ts",
+ "eval": "tsx src/scripts/run-evals.ts",
"prepare": "husky"
},
"dependencies": {
@@ -72,7 +73,8 @@
"remark-gfm": "^4.0.1",
"sonner": "^2.0.7",
"tailwind-merge": "^3.3.1",
- "vaul": "^1.1.2"
+ "vaul": "^1.1.2",
+ "zod": "^3.23.8"
},
"devDependencies": {
"@biomejs/biome": "2.2.0",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f314661..2305a41 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -150,7 +150,7 @@ importers:
version: 0.4.6(react-dom@19.1.1(react@19.1.1))(react@19.1.1)
openai:
specifier: ^5.15.0
- version: 5.20.2(ws@8.18.3)
+ version: 5.20.2(ws@8.18.3)(zod@3.25.76)
react:
specifier: 19.1.1
version: 19.1.1
@@ -184,6 +184,9 @@ importers:
vaul:
specifier: ^1.1.2
version: 1.1.2(@types/react-dom@19.1.9(@types/react@19.1.13))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)
+ zod:
+ specifier: ^3.23.8
+ version: 3.25.76
devDependencies:
'@biomejs/biome':
specifier: 2.2.0
@@ -9066,6 +9069,9 @@ packages:
resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==}
engines: {node: '>=12'}
+ zod@3.25.76:
+ resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==}
+
zwitch@2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
@@ -17054,9 +17060,10 @@ snapshots:
is-docker: 2.2.1
is-wsl: 2.2.0
- openai@5.20.2(ws@8.18.3):
+ openai@5.20.2(ws@8.18.3)(zod@3.25.76):
optionalDependencies:
ws: 8.18.3
+ zod: 3.25.76
openid-client@5.7.1:
dependencies:
@@ -20102,4 +20109,6 @@ snapshots:
y18n: 5.0.8
yargs-parser: 21.1.1
+ zod@3.25.76: {}
+
zwitch@2.0.4: {}
diff --git a/src/app/[id]/edit/page.tsx b/src/app/[id]/edit/page.tsx
index 8e9f4b3..bd2e505 100644
--- a/src/app/[id]/edit/page.tsx
+++ b/src/app/[id]/edit/page.tsx
@@ -100,20 +100,6 @@ export default async function EditBillPage({ params }: Params) {
className="w-full min-h-32 border rounded p-2"
/>
-
-
-
-
@@ -154,13 +125,12 @@ export async function generateMetadata(
const q = sp?.q;
const title = id;
const description = `Bill ${id} analysis and judgement`;
- const h = headers();
- const host = (await h).get("x-forwarded-host") || (await h).get("host") || "";
- const proto = ((await h).get("x-forwarded-proto") || "https").split(",")[0];
// Ensure we always have a base URL for absolute image URLs (required for Twitter Cards)
const baseUrl =
env.NEXT_PUBLIC_APP_URL ||
- (host ? `${proto}://${host}` : "http://localhost:3000");
+ (env.NODE_ENV === "production"
+ ? BUILD_CANADA_URL
+ : "http://localhost:3000");
const pagePath = buildRelativePath(id);
const pageUrl = `${baseUrl}${pagePath}`;
const pageUrlWithQuery = q
diff --git a/src/app/api/[id]/reprocess/route.ts b/src/app/api/[id]/reprocess/route.ts
index 63a50ad..7e217d1 100644
--- a/src/app/api/[id]/reprocess/route.ts
+++ b/src/app/api/[id]/reprocess/route.ts
@@ -89,7 +89,17 @@ export async function POST(
);
}
- const analysis = await summarizeBillText(markdown);
+ let analysis: Awaited>;
+ try {
+ analysis = await summarizeBillText(markdown);
+ } catch (error) {
+ // Never overwrite a good stored analysis with a failure stub.
+ console.error(`Reprocess ${id}: AI analysis failed`, error);
+ return NextResponse.json(
+ { error: "AI analysis failed; existing analysis left unchanged" },
+ { status: 502 },
+ );
+ }
const latestStageDate =
apiBill.stages && apiBill.stages.length > 0
@@ -123,10 +133,9 @@ export async function POST(
tenet_evaluations: analysis.tenet_evaluations,
final_judgment: analysis.final_judgment,
rationale: analysis.rationale,
- needs_more_info: analysis.needs_more_info,
- missing_details: analysis.missing_details,
steel_man: analysis.steel_man,
question_period_questions: analysis.question_period_questions ?? [],
+ isSocialIssue: analysis.is_social_issue === "yes",
lastUpdatedOn: new Date(latestStageDate),
},
},
diff --git a/src/app/api/[id]/route.ts b/src/app/api/[id]/route.ts
index e953709..15a5f9b 100644
--- a/src/app/api/[id]/route.ts
+++ b/src/app/api/[id]/route.ts
@@ -33,10 +33,8 @@ export async function POST(
let final_judgment: string | undefined;
let rationale: string | undefined;
let steel_man: string | undefined;
- let missing_details_input: unknown;
let genres_input: unknown;
let question_period_questions_input: unknown;
- let hasMissingDetails = false;
let hasGenres = false;
let hasQuestionPeriodQuestions = false;
let tenet_ids: string[] = [];
@@ -53,10 +51,6 @@ export async function POST(
final_judgment = params.get("final_judgment") || undefined;
rationale = params.get("rationale") || undefined;
steel_man = params.get("steel_man") || undefined;
- if (params.has("missing_details")) {
- hasMissingDetails = true;
- missing_details_input = params.get("missing_details") || "";
- }
if (params.has("genres")) {
hasGenres = true;
genres_input = params.get("genres") || "";
@@ -80,10 +74,6 @@ export async function POST(
final_judgment = asString(json.final_judgment);
rationale = asString(json.rationale);
steel_man = asString(json.steel_man);
- if ("missing_details" in json) {
- hasMissingDetails = true;
- missing_details_input = (json as any).missing_details;
- }
if ("genres" in json) {
hasGenres = true;
genres_input = (json as any).genres;
@@ -115,11 +105,6 @@ export async function POST(
(form.get("final_judgment") as string | null) || undefined;
rationale = (form.get("rationale") as string | null) || undefined;
steel_man = (form.get("steel_man") as string | null) || undefined;
- if (typeof form.has === "function" && form.has("missing_details")) {
- hasMissingDetails = true;
- missing_details_input =
- (form.get("missing_details") as string | null) || "";
- }
if (typeof form.has === "function" && form.has("genres")) {
hasGenres = true;
genres_input = (form.get("genres") as string | null) || "";
@@ -259,11 +244,6 @@ export async function POST(
return questions.map((question) => ({ question }));
};
- let missing_details: string[] | undefined;
- if (hasMissingDetails) {
- missing_details = parseCommaSeparated(missing_details_input);
- }
-
let genres: string[] | undefined;
if (hasGenres) {
genres = parseCommaSeparated(genres_input);
@@ -285,9 +265,6 @@ export async function POST(
if (final_judgment !== undefined) update.final_judgment = final_judgment;
if (rationale !== undefined) update.rationale = rationale;
if (steel_man !== undefined) update.steel_man = steel_man;
- if (missing_details !== undefined) {
- update.missing_details = missing_details;
- }
if (genres !== undefined) {
update.genres = genres;
}
diff --git a/src/app/page.tsx b/src/app/page.tsx
index 317e52e..0c12feb 100644
--- a/src/app/page.tsx
+++ b/src/app/page.tsx
@@ -2,8 +2,6 @@ import { BillSummary } from "./types";
import BillExplorer from "./BillExplorer";
import { getAllBillsFromDB } from "@/server/get-all-bills-from-db";
import { fromBuildCanadaDbBill } from "@/utils/billConverters";
-import { getParliament45Header } from "@/components/BillDetail/BillHeader";
-import Markdown from "react-markdown";
import type { Metadata } from "next";
import { headers } from "next/headers";
import { env } from "@/env";
@@ -137,8 +135,6 @@ async function getMergedBills(): Promise {
isSocialIssue: dbBill.isSocialIssue,
final_judgment: dbBill.final_judgment as BillSummary["final_judgment"],
rationale: dbBill.rationale,
- needs_more_info: dbBill.needs_more_info,
- missing_details: dbBill.missing_details,
genres: dbBill.genres,
parliamentNumber: dbBill.parliamentNumber,
sessionNumber: dbBill.sessionNumber,
@@ -173,8 +169,6 @@ async function getMergedBills(): Promise {
isSocialIssue: dbBill.isSocialIssue,
final_judgment: dbBill.final_judgment as BillSummary["final_judgment"],
rationale: dbBill.rationale,
- needs_more_info: dbBill.needs_more_info,
- missing_details: dbBill.missing_details,
genres: dbBill.genres,
parliamentNumber: dbBill.parliamentNumber,
sessionNumber: dbBill.sessionNumber,
@@ -225,8 +219,6 @@ export default async function Home({
- {getParliament45Header()}
-
diff --git a/src/app/types.ts b/src/app/types.ts
index 22897bb..5ad4cce 100644
--- a/src/app/types.ts
+++ b/src/app/types.ts
@@ -31,8 +31,6 @@ export interface BillSummary {
summary?: string;
final_judgment?: "yes" | "no" | "abstain";
rationale?: string;
- needs_more_info?: boolean;
- missing_details?: string[];
genres?: string[];
tenet_evaluations?: Array<{
id?: number;
diff --git a/src/components/BillDetail/BillHeader.tsx b/src/components/BillDetail/BillHeader.tsx
index 133e82f..7b953b3 100644
--- a/src/components/BillDetail/BillHeader.tsx
+++ b/src/components/BillDetail/BillHeader.tsx
@@ -4,37 +4,6 @@ interface BillHeaderProps {
bill: UnifiedBill;
}
-export function getParliament45Header() {
- const termStart = new Date("2025-05-26T00:00:00-04:00"); // Opening of the 45th Parliament (1st session)
- const today = new Date();
- const msPerDay = 24 * 60 * 60 * 1000;
- const _daysSinceStart = Math.floor(
- (today.getTime() - termStart.getTime()) / msPerDay,
- );
- // const daySinceStartSuffix =
-
- // const primeMinister = "Mark Carney"; // PM since Mar 14, 2025
- // const oppositionLeader = "Pierre Poilievre"; // Official Opposition Leader since Aug 18, 2025
-
- // // House of Commons party standings at opening of the 45th Parliament
- // const seats = {
- // Liberal: 169,
- // Conservative: 144,
- // "Bloc Québécois": 22,
- // NDP: 7,
- // Green: 1,
- // };
-
- // // const seatList = Object.entries(seats)
- // .map(([party, count]) => `${party} ${count}`)
- // .join(" • ");
-
- // return `It's is the
- // **${daysSinceStart}**${daySinceStartSuffix}
- // day of the 45th Canadian Parliament began.
- // `;
-}
-
export function BillHeader({ bill }: BillHeaderProps) {
return (