From 7381852dee15e9c8704e7961972329cc1e4ded01 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 14:15:06 -0700 Subject: [PATCH] feat: add caselaw benchmark snapshot --- CASELAW_METHODOLOGY.md | 23 ++ METHODOLOGY.md | 19 +- README.md | 13 +- data/case-authorities-v2.json | 9 + exports/latest.json | 580 +++++++++++++++++++++++++++---- results/v2.0.0/README.md | 14 + results/v2.0.0/manifest.json | 584 ++++++++++++++++++++++++++++++++ scripts/build_holdout_v2.py | 30 ++ scripts/publish_v2_snapshot.py | 91 +++++ src/evidencebench/citations.py | 52 ++- src/evidencebench/runner.py | 10 +- src/evidencebench/scoring.py | 21 +- src/evidencebench/validation.py | 2 +- tests/test_citations.py | 7 + tests/test_scoring.py | 28 ++ 15 files changed, 1395 insertions(+), 88 deletions(-) create mode 100644 CASELAW_METHODOLOGY.md create mode 100644 data/case-authorities-v2.json create mode 100644 results/v2.0.0/README.md create mode 100644 results/v2.0.0/manifest.json create mode 100644 scripts/build_holdout_v2.py create mode 100644 scripts/publish_v2_snapshot.py diff --git a/CASELAW_METHODOLOGY.md b/CASELAW_METHODOLOGY.md new file mode 100644 index 0000000..db7d24e --- /dev/null +++ b/CASELAW_METHODOLOGY.md @@ -0,0 +1,23 @@ +# EvidenceBench caselaw module + +The v2 sealed holdout adds 16 evidence-caselaw questions to the 96 FRE-focused +items. The caselaw module is balanced across eight intersections: + +- federal and state authority; +- civil and criminal proceedings; and +- widely cited and less familiar decisions. + +Each intersection contains two questions. Popularity is a sampling stratum, +not a claim about precedential weight. The questions test holdings concerning +expert gatekeeping, scientific evidence, eyewitness identification, hearsay, +prior statements, and Rule 403. + +Case existence and citation form are validated against a frozen reporter index. +Question rationales were checked against official court publications or official +reporters where available. The benchmark requires a reporter citation; a case +name without a reporter citation does not satisfy citation recall. + +The v2 prompt asks for legal authorities. FRE questions continue to require +normalized FRE citations, while caselaw questions require official reporter +citations. Answer accuracy, authority precision, authority recall, authority +F1, hallucination rate, and the 70/30 overall formula are otherwise unchanged. diff --git a/METHODOLOGY.md b/METHODOLOGY.md index 738fcba..8dfc4aa 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -1,16 +1,16 @@ # EvidenceBench v1 methodology -EvidenceBench evaluates closed-book responses to attorney-authored Federal -Rules of Evidence questions. A model receives a multiple-choice question and -must return JSON containing one option ID, a concise explanation, and Federal -Rules of Evidence citations. No browsing, retrieval, tools, or external -authority text is available to the model. +EvidenceBench evaluates closed-book responses to attorney-authored evidence +questions. A model receives a multiple-choice question and must return JSON +containing one option ID, a concise explanation, and supporting FRE or official +reporter citations. No browsing, retrieval, tools, or external authority text +is available to the model. ## Dataset - Development set: 24 public questions licensed CC BY-NC 4.0. -- Official holdout: 96 newly authored sealed questions; only aggregate results - are published. +- Official v2 holdout: 112 sealed questions—96 FRE-focused and 16 caselaw; + only aggregate results are published. - Every item has a stable ID, category, difficulty, rationale, gold answer, required citation groups, accepted citations, corpus version, and attorney review record. @@ -21,8 +21,9 @@ Answer accuracy is exact option match. Citation precision is accepted cited rules divided by all cited rules. Citation recall is the share of required citation groups satisfied. Citation F1 is their harmonic mean. A citation that does not exist in the frozen FRE corpus is hallucinated; a real but -unannotated rule is unsupported. Overall score is 70% answer accuracy and 30% -citation F1. +unannotated authority is unsupported. Overall score is 70% answer accuracy and +30% authority-citation F1. Caselaw results are additionally aggregated across +federal/state, civil/criminal, and popular/obscure dimensions. ## Reproducibility diff --git a/README.md b/README.md index 2dddbb3..76f08f5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # EvidenceBench EvidenceBench is Objection Academy's public benchmark for evidence-law answer -accuracy and Federal Rules of Evidence citation reliability. It is an +accuracy and legal-authority citation reliability. It is an educational research project, not legal advice and not a substitute for professional legal research. @@ -10,11 +10,14 @@ professional legal research. - A 24-question development set and its scoring annotations. - The prompt contract, scoring code, model manifests, methodology, and website export format. -- Aggregate official results after an attorney-approved sealed holdout run. +- Aggregate official results after an approved sealed holdout run, including + separate FRE and caselaw dimensions. ## What is sealed -The 96-question official holdout will be newly authored, attorney-reviewed, +The v2 official holdout contains 96 FRE-focused questions plus a 16-question +caselaw module balanced across federal/state, civil/criminal, and +popular/obscure decisions. It is attorney-reviewed, and never distributed with the Objection Academy app. Its prompts, annotations, and item-level results are not public. Once approved, published manifests will commit to the holdout with a SHA-256 hash, counts, categories, @@ -33,11 +36,11 @@ credentials named in that manifest. It never enables web search or tools. ## Metrics -Each item receives answer accuracy, citation precision, citation recall, +Each item receives answer accuracy, authority precision, authority recall, citation F1, citation-existence status, and unsupported-citation status. Official overall score is `0.70 * answer accuracy + 0.30 * citation F1`. Missing citations, refusals, and invalid structured outputs score zero for -citation F1. +authority F1. Caselaw questions require an official reporter citation. ## Authority corpus diff --git a/data/case-authorities-v2.json b/data/case-authorities-v2.json new file mode 100644 index 0000000..af445bd --- /dev/null +++ b/data/case-authorities-v2.json @@ -0,0 +1,9 @@ +{ + "version": "CASELAW-2026-07-14", + "citations": [ + "509 U.S. 579", "526 U.S. 137", "528 U.S. 440", "552 U.S. 379", + "541 U.S. 36", "519 U.S. 172", "513 U.S. 150", "505 U.S. 317", + "55 Cal. 4th 747", "7 N.Y.3d 434", "6 N.Y.3d 636", "234 N.J. 340", + "17 Cal. 3d 24", "208 N.J. 208", "352 Or. 724", "92 A.3d 766" + ] +} diff --git a/exports/latest.json b/exports/latest.json index 0240462..2c17520 100644 --- a/exports/latest.json +++ b/exports/latest.json @@ -1,25 +1,36 @@ { "benchmark_name": "EvidenceBench", - "benchmark_version": "1.0.0", + "benchmark_version": "2.0.0", "status": "official_snapshot", - "published_at": "2026-07-14T00:00:00Z", + "published_at": "2026-07-14T21:13:47Z", "authority_corpus": { - "version": "FRE-2025-12-01", - "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf", + "version": "FRE-2025-12-01+CASELAW-2026-07-14", + "fre_source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf", + "case_index_version": "CASELAW-2026-07-14", "pending_amendments_excluded": [ { "rule": "FRE 801(d)(1)(A)", - "effective_date": "2026-12-01", - "status_as_of": "2026-07-13", - "status": "transmitted_to_congress_pending_effectiveness" + "effective_date": "2026-12-01" } ] }, "holdout": { "status": "sealed", - "question_count": 96, + "question_count": 112, + "fre_question_count": 96, + "caselaw_question_count": 16, "development_question_count": 24, - "dataset_sha256": "06c5f639679f6d06995403a910bfe2a40dd7ca517400e7c8c291d7f360331cba", + "dataset_sha256": "13e319b3c332f4ede8d5b34fcc35a1a53db845626f1982bde3e622231859aea7", + "caselaw_matrix": { + "federal_civil_popular": 2, + "federal_civil_obscure": 2, + "federal_criminal_popular": 2, + "federal_criminal_obscure": 2, + "state_civil_popular": 2, + "state_civil_obscure": 2, + "state_criminal_popular": 2, + "state_criminal_obscure": 2 + }, "categories": [ "relevance", "hearsay", @@ -29,57 +40,194 @@ "best_evidence", "character", "impeachment", - "privilege" + "privilege", + "caselaw" ] }, "protocol": { - "name": "evidencebench-v1-closed-book", + "name": "evidencebench-v2-closed-book", + "prompt_version": "evidencebench-v2-closed-book", + "prompt_sha256": "55f6eaba94060b280ef31c6a009618f4c98bb2ce336f607c82b0c49fb40a147a", "runs_per_question": 1, "tools_enabled": false, "overall_formula": "0.70 * answer_accuracy + 0.30 * citation_f1" }, "models": [ { - "display_name": "GPT-5.6 Sol", - "family": "OpenAI", - "model_id": "openai/gpt-5.6-sol", + "display_name": "Gemini 3.5 Flash", + "family": "Google", + "model_id": "google/gemini-3.5-flash", "provider_route": "OpenRouter pinned route", "status": "official", - "overall": 0.9844, + "overall": 0.9643, "answer_accuracy": 1.0, - "citation_f1": 0.9479, - "hallucination_rate": 0.0091, - "cost_usd": 0.25238, - "input_tokens": 14920, - "output_tokens": 5926 + "citation_f1": 0.881, + "hallucination_rate": 0.0079, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 0.9792, + "count": 16, + "hallucination_rate": 0.0588, + "overall": 0.9937, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 1.0, + "citation_f1": 0.8646, + "count": 96, + "hallucination_rate": 0.0, + "overall": 0.9594, + "unsupported_citation_count": 15 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.132404, + "input_tokens": 20931, + "output_tokens": 11223 }, { - "display_name": "Mistral Large", + "display_name": "Mistral Large 3", "family": "Mistral", "model_id": "mistralai/mistral-large-2512", "provider_route": "OpenRouter pinned route", "status": "official", - "overall": 0.974, - "answer_accuracy": 1.0, - "citation_f1": 0.9132, - "hallucination_rate": 0.0339, - "cost_usd": 0.017155, - "input_tokens": 15252, - "output_tokens": 8011 - }, - { - "display_name": "Gemini 3.5 Flash", - "family": "Google", - "model_id": "google/gemini-3.5-flash", - "provider_route": "OpenRouter pinned route", - "status": "official", - "overall": 0.9727, + "overall": 0.9641, "answer_accuracy": 1.0, - "citation_f1": 0.909, - "hallucination_rate": 0.0, - "cost_usd": 0.096867, - "input_tokens": 14988, - "output_tokens": 8265 + "citation_f1": 0.8804, + "hallucination_rate": 0.0543, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 0.8125, + "count": 16, + "hallucination_rate": 0.1111, + "overall": 0.9437, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 1.0, + "citation_f1": 0.8917, + "count": 96, + "hallucination_rate": 0.045, + "overall": 0.9675, + "unsupported_citation_count": 9 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.925, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9625, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 0.625, + "count": 8, + "hallucination_rate": 0.2857, + "overall": 0.8875, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.1, + "overall": 0.925, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.125, + "overall": 0.9625, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.022883, + "input_tokens": 21011, + "output_tokens": 10797 }, { "display_name": "Grok 4.5", @@ -87,13 +235,87 @@ "model_id": "x-ai/grok-4.5", "provider_route": "OpenRouter pinned route", "status": "official", - "overall": 0.9656, + "overall": 0.942, "answer_accuracy": 1.0, - "citation_f1": 0.8854, - "hallucination_rate": 0.0226, - "cost_usd": 0.30921, - "input_tokens": 36192, - "output_tokens": 44303 + "citation_f1": 0.8065, + "hallucination_rate": 0.0, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 16, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 1.0, + "citation_f1": 0.7743, + "count": 96, + "hallucination_rate": 0.0, + "overall": 0.9323, + "unsupported_citation_count": 25 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.404352, + "input_tokens": 45468, + "output_tokens": 57516 }, { "display_name": "DeepSeek V4 Pro", @@ -101,13 +323,175 @@ "model_id": "deepseek/deepseek-v4-pro", "provider_route": "OpenRouter pinned route", "status": "official", - "overall": 0.9458, - "answer_accuracy": 0.9688, - "citation_f1": 0.8924, - "hallucination_rate": 0.0101, - "cost_usd": 0.092513, - "input_tokens": 15731, - "output_tokens": 34259 + "overall": 0.9289, + "answer_accuracy": 0.9554, + "citation_f1": 0.8673, + "hallucination_rate": 0.0084, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 0.9375, + "citation_f1": 0.9375, + "count": 16, + "hallucination_rate": 0.0, + "overall": 0.9375, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 0.9583, + "citation_f1": 0.8556, + "count": 96, + "hallucination_rate": 0.0096, + "overall": 0.9275, + "unsupported_citation_count": 10 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.102966, + "input_tokens": 20681, + "output_tokens": 39696 + }, + { + "display_name": "GPT-5.6 Sol", + "family": "OpenAI", + "model_id": "openai/gpt-5.6-sol", + "provider_route": "OpenRouter pinned route", + "status": "official", + "overall": 0.9268, + "answer_accuracy": 0.9464, + "citation_f1": 0.881, + "hallucination_rate": 0.0, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 0.875, + "citation_f1": 0.8125, + "count": 16, + "hallucination_rate": 0.0, + "overall": 0.8562, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 0.9583, + "citation_f1": 0.8924, + "count": 96, + "hallucination_rate": 0.0, + "overall": 0.9385, + "unsupported_citation_count": 3 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 0.75, + "citation_f1": 0.625, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.7125, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 0.875, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.8375, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 0.75, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.75, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.9625, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.329485, + "input_tokens": 19295, + "output_tokens": 7767 }, { "display_name": "Claude Fable 5", @@ -115,13 +499,87 @@ "model_id": "anthropic/claude-fable-5", "provider_route": "OpenRouter pinned route", "status": "official", - "overall": 0.9367, - "answer_accuracy": 1.0, - "citation_f1": 0.7889, - "hallucination_rate": 0.0151, - "cost_usd": 1.12072, - "input_tokens": 24252, - "output_tokens": 17564 + "overall": 0.9237, + "answer_accuracy": 0.9643, + "citation_f1": 0.8292, + "hallucination_rate": 0.0457, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 0.9792, + "count": 16, + "hallucination_rate": 0.0476, + "overall": 0.9937, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 0.9583, + "citation_f1": 0.8042, + "count": 96, + "hallucination_rate": 0.0455, + "overall": 0.9121, + "unsupported_citation_count": 34 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.0909, + "overall": 0.9875, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.0769, + "overall": 0.9875, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 1.33058, + "input_tokens": 31548, + "output_tokens": 20302 } ], "methodology_url": "https://github.com/Examp-LLC/EvidenceBench/blob/main/METHODOLOGY.md", diff --git a/results/v2.0.0/README.md b/results/v2.0.0/README.md new file mode 100644 index 0000000..e26a7fa --- /dev/null +++ b/results/v2.0.0/README.md @@ -0,0 +1,14 @@ +# EvidenceBench v2.0.0 snapshot + +This snapshot evaluates six pinned OpenRouter model routes on a sealed +112-question holdout: 96 FRE-focused questions and 16 evidence-caselaw +questions. The caselaw module contains two questions in every intersection of +federal/state, civil/criminal, and popular/obscure decisions. + +Only aggregate results are public. Sealed prompts, annotations, and item-level +model outputs are excluded from git. The manifest records the holdout +commitment, prompt hash, model IDs, token usage, cost, overall metrics, and +dimension-level metrics. + +v2 scores are not directly comparable to v1 because the prompt contract, +authority corpus, and holdout composition changed. diff --git a/results/v2.0.0/manifest.json b/results/v2.0.0/manifest.json new file mode 100644 index 0000000..3cae0c6 --- /dev/null +++ b/results/v2.0.0/manifest.json @@ -0,0 +1,584 @@ +{ + "benchmark_version": "2.0.0", + "status": "official_snapshot", + "published_at": "2026-07-14T21:13:47Z", + "authority_corpus": { + "version": "FRE-2025-12-01+CASELAW-2026-07-14", + "fre_source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf", + "case_index_version": "CASELAW-2026-07-14", + "pending_amendments_excluded": [ + { + "rule": "FRE 801(d)(1)(A)", + "effective_date": "2026-12-01" + } + ] + }, + "holdout": { + "status": "sealed", + "question_count": 112, + "fre_question_count": 96, + "caselaw_question_count": 16, + "development_question_count": 24, + "dataset_sha256": "13e319b3c332f4ede8d5b34fcc35a1a53db845626f1982bde3e622231859aea7", + "caselaw_matrix": { + "federal_civil_popular": 2, + "federal_civil_obscure": 2, + "federal_criminal_popular": 2, + "federal_criminal_obscure": 2, + "state_civil_popular": 2, + "state_civil_obscure": 2, + "state_criminal_popular": 2, + "state_criminal_obscure": 2 + }, + "categories": [ + "relevance", + "hearsay", + "witnesses", + "experts", + "authentication", + "best_evidence", + "character", + "impeachment", + "privilege", + "caselaw" + ] + }, + "protocol": { + "name": "evidencebench-v2-closed-book", + "prompt_version": "evidencebench-v2-closed-book", + "prompt_sha256": "55f6eaba94060b280ef31c6a009618f4c98bb2ce336f607c82b0c49fb40a147a", + "runs_per_question": 1, + "tools_enabled": false, + "overall_formula": "0.70 * answer_accuracy + 0.30 * citation_f1" + }, + "models": [ + { + "display_name": "Gemini 3.5 Flash", + "family": "Google", + "model_id": "google/gemini-3.5-flash", + "provider_route": "OpenRouter pinned route", + "status": "official", + "overall": 0.9643, + "answer_accuracy": 1.0, + "citation_f1": 0.881, + "hallucination_rate": 0.0079, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 0.9792, + "count": 16, + "hallucination_rate": 0.0588, + "overall": 0.9937, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 1.0, + "citation_f1": 0.8646, + "count": 96, + "hallucination_rate": 0.0, + "overall": 0.9594, + "unsupported_citation_count": 15 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.132404, + "input_tokens": 20931, + "output_tokens": 11223 + }, + { + "display_name": "Mistral Large 3", + "family": "Mistral", + "model_id": "mistralai/mistral-large-2512", + "provider_route": "OpenRouter pinned route", + "status": "official", + "overall": 0.9641, + "answer_accuracy": 1.0, + "citation_f1": 0.8804, + "hallucination_rate": 0.0543, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 0.8125, + "count": 16, + "hallucination_rate": 0.1111, + "overall": 0.9437, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 1.0, + "citation_f1": 0.8917, + "count": 96, + "hallucination_rate": 0.045, + "overall": 0.9675, + "unsupported_citation_count": 9 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.925, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.1111, + "overall": 0.9625, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 0.625, + "count": 8, + "hallucination_rate": 0.2857, + "overall": 0.8875, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.1, + "overall": 0.925, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.125, + "overall": 0.9625, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.022883, + "input_tokens": 21011, + "output_tokens": 10797 + }, + { + "display_name": "Grok 4.5", + "family": "xAI", + "model_id": "x-ai/grok-4.5", + "provider_route": "OpenRouter pinned route", + "status": "official", + "overall": 0.942, + "answer_accuracy": 1.0, + "citation_f1": 0.8065, + "hallucination_rate": 0.0, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 16, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 1.0, + "citation_f1": 0.7743, + "count": 96, + "hallucination_rate": 0.0, + "overall": 0.9323, + "unsupported_citation_count": 25 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.404352, + "input_tokens": 45468, + "output_tokens": 57516 + }, + { + "display_name": "DeepSeek V4 Pro", + "family": "DeepSeek", + "model_id": "deepseek/deepseek-v4-pro", + "provider_route": "OpenRouter pinned route", + "status": "official", + "overall": 0.9289, + "answer_accuracy": 0.9554, + "citation_f1": 0.8673, + "hallucination_rate": 0.0084, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 0.9375, + "citation_f1": 0.9375, + "count": 16, + "hallucination_rate": 0.0, + "overall": 0.9375, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 0.9583, + "citation_f1": 0.8556, + "count": 96, + "hallucination_rate": 0.0096, + "overall": 0.9275, + "unsupported_citation_count": 10 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.102966, + "input_tokens": 20681, + "output_tokens": 39696 + }, + { + "display_name": "GPT-5.6 Sol", + "family": "OpenAI", + "model_id": "openai/gpt-5.6-sol", + "provider_route": "OpenRouter pinned route", + "status": "official", + "overall": 0.9268, + "answer_accuracy": 0.9464, + "citation_f1": 0.881, + "hallucination_rate": 0.0, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 0.875, + "citation_f1": 0.8125, + "count": 16, + "hallucination_rate": 0.0, + "overall": 0.8562, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 0.9583, + "citation_f1": 0.8924, + "count": 96, + "hallucination_rate": 0.0, + "overall": 0.9385, + "unsupported_citation_count": 3 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 0.75, + "citation_f1": 0.625, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.7125, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 0.875, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.875, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 0.875, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.8375, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 0.75, + "citation_f1": 0.75, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.75, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.875, + "count": 8, + "hallucination_rate": 0.0, + "overall": 0.9625, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 0.329485, + "input_tokens": 19295, + "output_tokens": 7767 + }, + { + "display_name": "Claude Fable 5", + "family": "Anthropic", + "model_id": "anthropic/claude-fable-5", + "provider_route": "OpenRouter pinned route", + "status": "official", + "overall": 0.9237, + "answer_accuracy": 0.9643, + "citation_f1": 0.8292, + "hallucination_rate": 0.0457, + "dimensions": { + "domain": { + "caselaw": { + "answer_accuracy": 1.0, + "citation_f1": 0.9792, + "count": 16, + "hallucination_rate": 0.0476, + "overall": 0.9937, + "unsupported_citation_count": 0 + }, + "fre": { + "answer_accuracy": 0.9583, + "citation_f1": 0.8042, + "count": 96, + "hallucination_rate": 0.0455, + "overall": 0.9121, + "unsupported_citation_count": 34 + } + }, + "familiarity": { + "obscure": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.0909, + "overall": 0.9875, + "unsupported_citation_count": 0 + }, + "popular": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "jurisdiction": { + "federal": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.0769, + "overall": 0.9875, + "unsupported_citation_count": 0 + }, + "state": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + } + }, + "proceeding": { + "civil": { + "answer_accuracy": 1.0, + "citation_f1": 1.0, + "count": 8, + "hallucination_rate": 0.0, + "overall": 1.0, + "unsupported_citation_count": 0 + }, + "criminal": { + "answer_accuracy": 1.0, + "citation_f1": 0.9583, + "count": 8, + "hallucination_rate": 0.1, + "overall": 0.9875, + "unsupported_citation_count": 0 + } + } + }, + "cost_usd": 1.33058, + "input_tokens": 31548, + "output_tokens": 20302 + } + ] +} diff --git a/scripts/build_holdout_v2.py b/scripts/build_holdout_v2.py new file mode 100644 index 0000000..976f009 --- /dev/null +++ b/scripts/build_holdout_v2.py @@ -0,0 +1,30 @@ +"""Build the sealed v2 holdout and print its public commitment metadata.""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from pathlib import Path + + +def main() -> None: + sources = [Path("private/holdout-v1.jsonl"), Path("private/caselaw-v2.jsonl")] + rows = [json.loads(line) for source in sources for line in source.read_text().splitlines() if line] + ids = [row["id"] for row in rows] + if len(ids) != len(set(ids)): + raise ValueError("duplicate holdout IDs") + payload = "\n".join(json.dumps(row, sort_keys=True) for row in rows) + "\n" + destination = Path("private/holdout-v2.jsonl") + destination.write_text(payload) + matrix = Counter(row["category"] for row in rows if row["category"].startswith("caselaw_")) + print(json.dumps({ + "question_count": len(rows), + "caselaw_question_count": sum(matrix.values()), + "caselaw_matrix": dict(sorted(matrix.items())), + "dataset_sha256": hashlib.sha256(payload.encode()).hexdigest(), + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/publish_v2_snapshot.py b/scripts/publish_v2_snapshot.py new file mode 100644 index 0000000..74e8541 --- /dev/null +++ b/scripts/publish_v2_snapshot.py @@ -0,0 +1,91 @@ +"""Create aggregate public artifacts from validated private v2 runs.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +from datetime import UTC, datetime +from pathlib import Path + +from evidencebench.datasets import file_sha256 +from evidencebench.runner import PROMPT_VERSION, prompt_for + + +MODELS = { + "openai-gpt-5-6-sol": ("OpenAI", "models/openai-gpt-5-6-sol.json"), + "anthropic-claude-fable-5": ("Anthropic", "models/anthropic-claude-fable-5.json"), + "google-gemini-3-5-flash": ("Google", "models/google-gemini-3-5-flash.json"), + "xai-grok-4-5": ("xAI", "models/xai-grok-4-5.json"), + "deepseek-v4-pro": ("DeepSeek", "models/deepseek-v4-pro.json"), + "mistral-large-3": ("Mistral", "models/mistral-large-3.json"), +} + + +def main() -> None: + run_dir = Path("private/runs/v2.0.0") + models = [] + for slug, (family, manifest_path) in MODELS.items(): + manifest = json.loads(Path(manifest_path).read_text()) + responses = [json.loads(line) for line in (run_dir / f"{slug}.jsonl").read_text().splitlines() if line] + ids = [response["question_id"] for response in responses] + if len(responses) != 112 or len(set(ids)) != 112: + raise ValueError(f"{slug}: expected 112 unique responses") + score = json.loads((run_dir / f"{slug}.score.json").read_text()) + usage = [response.get("usage") or {} for response in responses] + models.append({ + "display_name": manifest["display_name"], + "family": family, + "model_id": manifest["model_id"], + "provider_route": manifest["provider_route"], + "status": "official", + **{key: score["overall"][key] for key in ("overall", "answer_accuracy", "citation_f1", "hallucination_rate")}, + "dimensions": score["dimensions"], + "cost_usd": round(sum(item.get("cost", 0) or 0 for item in usage), 6), + "input_tokens": sum(item.get("prompt_tokens", 0) or 0 for item in usage), + "output_tokens": sum(item.get("completion_tokens", 0) or 0 for item in usage), + }) + models.sort(key=lambda model: model["overall"], reverse=True) + published_at = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + manifest = { + "benchmark_version": "2.0.0", + "status": "official_snapshot", + "published_at": published_at, + "authority_corpus": { + "version": "FRE-2025-12-01+CASELAW-2026-07-14", + "fre_source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf", + "case_index_version": "CASELAW-2026-07-14", + "pending_amendments_excluded": [{"rule": "FRE 801(d)(1)(A)", "effective_date": "2026-12-01"}], + }, + "holdout": { + "status": "sealed", + "question_count": 112, + "fre_question_count": 96, + "caselaw_question_count": 16, + "development_question_count": 24, + "dataset_sha256": file_sha256("private/holdout-v2.jsonl"), + "caselaw_matrix": { + "federal_civil_popular": 2, "federal_civil_obscure": 2, + "federal_criminal_popular": 2, "federal_criminal_obscure": 2, + "state_civil_popular": 2, "state_civil_obscure": 2, + "state_criminal_popular": 2, "state_criminal_obscure": 2, + }, + "categories": ["relevance", "hearsay", "witnesses", "experts", "authentication", "best_evidence", "character", "impeachment", "privilege", "caselaw"], + }, + "protocol": { + "name": "evidencebench-v2-closed-book", + "prompt_version": PROMPT_VERSION, + "prompt_sha256": hashlib.sha256(inspect.getsource(prompt_for).encode()).hexdigest(), + "runs_per_question": 1, + "tools_enabled": False, + "overall_formula": "0.70 * answer_accuracy + 0.30 * citation_f1", + }, + "models": models, + } + destination = Path("results/v2.0.0/manifest.json") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(manifest, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/src/evidencebench/citations.py b/src/evidencebench/citations.py index 6c61313..725297e 100644 --- a/src/evidencebench/citations.py +++ b/src/evidencebench/citations.py @@ -10,28 +10,68 @@ re.IGNORECASE, ) +_CASE_CITATION = re.compile( + r"(?P\d+)\s+" + r"(?PU\.?\s*S\.?|Cal\.?\s*(?:3d|4th|5th)|N\.?\s*Y\.?\s*3d|" + r"N\.?\s*J\.?|Or\.?|A\.?\s*3d)\s+" + r"(?P\d+)", + re.IGNORECASE, +) + +_REPORTERS = { + "US": "U.S.", + "CAL3D": "Cal. 3d", + "CAL4TH": "Cal. 4th", + "CAL5TH": "Cal. 5th", + "NY3D": "N.Y.3d", + "NJ": "N.J.", + "OR": "Or.", + "A3D": "A.3d", +} + def _index_path() -> Path: return Path(__file__).resolve().parents[2] / "data" / "fre-2025-rules.json" +def _case_index_path() -> Path: + return Path(__file__).resolve().parents[2] / "data" / "case-authorities-v2.json" + + def load_rule_index() -> dict[str, list[str]]: return json.loads(_index_path().read_text())["rules"] +def load_case_index() -> set[str]: + return set(json.loads(_case_index_path().read_text())["citations"]) + + def normalize_citation(value: str) -> str | None: - match = _CITATION.search(value.strip()) - if not match: - return None - rule = match.group("rule") - subsections = match.group("subsections") or "" - return f"FRE {rule}{subsections.lower()}" + stripped = value.strip() + if re.search(r"(?:FRE|Fed(?:eral)?\.?\s*R(?:ule)?\.?\s*Evid\.?|Rule)\s*\d", stripped, re.I): + match = _CITATION.search(stripped) + if match: + rule = match.group("rule") + subsections = match.group("subsections") or "" + return f"FRE {rule}{subsections.lower()}" + case_match = _CASE_CITATION.search(stripped) + if case_match: + reporter_key = re.sub(r"[^A-Z0-9]", "", case_match.group("reporter").upper()) + reporter = _REPORTERS.get(reporter_key) + if reporter: + return f"{case_match.group('volume')} {reporter} {case_match.group('page')}" + match = _CITATION.fullmatch(stripped) + if match: + return f"FRE {match.group('rule')}{(match.group('subsections') or '').lower()}" + return None def citation_exists(citation: str, index: dict[str, list[str]] | None = None) -> bool: normalized = normalize_citation(citation) if not normalized: return False + if not normalized.startswith("FRE "): + return normalized in load_case_index() index = index or load_rule_index() match = re.fullmatch(r"FRE (\d{3,4})((?:\([a-z0-9]+\))*)", normalized) if not match: diff --git a/src/evidencebench/runner.py b/src/evidencebench/runner.py index bdf204b..79562f7 100644 --- a/src/evidencebench/runner.py +++ b/src/evidencebench/runner.py @@ -10,16 +10,18 @@ from .datasets import file_sha256, load_questions -PROMPT_VERSION = "evidencebench-v1-closed-book" +PROMPT_VERSION = "evidencebench-v2-closed-book" def prompt_for(question) -> str: options = "\n".join(f"{option['id']}. {option['text']}" for option in question.options) return f"""You are completing a closed-book Federal Rules of Evidence exercise. Do not browse, use tools, or claim to have consulted external material. -Choose the best answer and cite only Federal Rules of Evidence provisions that -support the answer. Return JSON only in this exact shape: -{{"choice_id":"A","explanation":"brief explanation","citations":["FRE 801(c)"]}} +Choose the best answer and cite only legal authorities that support it. For a +rule question, cite normalized Federal Rules of Evidence provisions. For a +caselaw question, cite the case's official reporter citation. Return JSON only +in this exact shape: +{{"choice_id":"A","explanation":"brief explanation","citations":["509 U.S. 579"]}} Question: {question.stem} diff --git a/src/evidencebench/scoring.py b/src/evidencebench/scoring.py index 0394d90..787e236 100644 --- a/src/evidencebench/scoring.py +++ b/src/evidencebench/scoring.py @@ -38,9 +38,19 @@ def score_item(question: Question, response: ModelResponse) -> ItemScore: def aggregate(scores: list[ItemScore], questions: list[Question]) -> dict: by_category: dict[str, list[ItemScore]] = defaultdict(list) + by_dimension: dict[str, dict[str, list[ItemScore]]] = defaultdict(lambda: defaultdict(list)) category_by_id = {question.id: question.category for question in questions} for score in scores: - by_category[category_by_id[score.question_id]].append(score) + category = category_by_id[score.question_id] + by_category[category].append(score) + if category.startswith("caselaw_"): + _, jurisdiction, proceeding, familiarity = category.split("_", 3) + by_dimension["domain"]["caselaw"].append(score) + by_dimension["jurisdiction"][jurisdiction].append(score) + by_dimension["proceeding"][proceeding].append(score) + by_dimension["familiarity"][familiarity].append(score) + else: + by_dimension["domain"]["fre"].append(score) def summarize(items: list[ItemScore]) -> dict: count = len(items) or 1 @@ -59,4 +69,11 @@ def summarize(items: list[ItemScore]) -> dict: "overall": round(0.70 * accuracy + 0.30 * citation_f1, 4), } - return {"overall": summarize(scores), "categories": {key: summarize(value) for key, value in sorted(by_category.items())}} + return { + "overall": summarize(scores), + "categories": {key: summarize(value) for key, value in sorted(by_category.items())}, + "dimensions": { + dimension: {key: summarize(value) for key, value in sorted(groups.items())} + for dimension, groups in sorted(by_dimension.items()) + }, + } diff --git a/src/evidencebench/validation.py b/src/evidencebench/validation.py index e8a1b22..efc9c17 100644 --- a/src/evidencebench/validation.py +++ b/src/evidencebench/validation.py @@ -17,7 +17,7 @@ def validate_questions(path: str) -> list[str]: option_ids = {option["id"] for option in question.options} if question.gold_choice_id not in option_ids: errors.append(f"{question.id}: gold choice is not an option") - if question.corpus_version != "FRE-2025-12-01": + if question.corpus_version not in {"FRE-2025-12-01", "FRE-2025-12-01+CASELAW-2026-07-14"}: errors.append(f"{question.id}: unexpected corpus version") if question.attorney_review_status not in {"APPROVED", "PENDING"}: errors.append(f"{question.id}: invalid review status") diff --git a/tests/test_citations.py b/tests/test_citations.py index a5f678f..a74cca1 100644 --- a/tests/test_citations.py +++ b/tests/test_citations.py @@ -12,3 +12,10 @@ def test_rejects_nonexistent_rules_and_subsections(self): self.assertTrue(citation_exists("FRE 801(c)")) self.assertFalse(citation_exists("FRE 801(z)")) self.assertFalse(citation_exists("FRE 999")) + + def test_normalizes_case_reporter_citations(self): + self.assertEqual(normalize_citation("Daubert, 509 U.S. 579 (1993)"), "509 U.S. 579") + self.assertEqual(normalize_citation("55 Cal.4th 747"), "55 Cal. 4th 747") + self.assertEqual(normalize_citation("Parker, 7 NY3d 434"), "7 N.Y.3d 434") + self.assertTrue(citation_exists("208 N.J. 208")) + self.assertFalse(citation_exists("999 U.S. 999")) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 0c5ecb2..23ffed7 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -36,7 +36,35 @@ def test_missing_and_invalid_output_score_zero(self): self.assertEqual(score.answer_accuracy, 0) self.assertEqual(score.citation_f1, 0) + def test_case_reporter_citation_scores_as_supported_authority(self): + question = Question( + **{ + **sample_question().__dict__, + "id": "EB-CASE-CITE", + "category": "caselaw_federal_civil_popular", + "citations": CitationAnnotation([["509 U.S. 579"]], ["509 U.S. 579", "FRE 702"]), + } + ) + score = score_item( + question, + ModelResponse(question.id, "A", "", ["Daubert, 509 US 579 (1993)"]), + ) + self.assertEqual(score.citation_f1, 1) + self.assertEqual(score.hallucinated_citations, []) + def test_aggregate_uses_overall_formula(self): question = sample_question() result = aggregate([score_item(question, ModelResponse("EB-T-001", "A", "", ["FRE 801(c)"]))], [question]) self.assertEqual(result["overall"]["overall"], 1) + self.assertEqual(result["dimensions"]["domain"]["fre"]["count"], 1) + + def test_aggregate_exposes_caselaw_dimensions(self): + question = Question( + **{**sample_question().__dict__, "id": "EB-CASE-T", "category": "caselaw_state_criminal_obscure"} + ) + score = score_item(question, ModelResponse(question.id, "A", "", ["FRE 801(c)"])) + result = aggregate([score], [question]) + self.assertEqual(result["dimensions"]["domain"]["caselaw"]["count"], 1) + self.assertEqual(result["dimensions"]["jurisdiction"]["state"]["count"], 1) + self.assertEqual(result["dimensions"]["proceeding"]["criminal"]["count"], 1) + self.assertEqual(result["dimensions"]["familiarity"]["obscure"]["count"], 1)