From 5c58589a7334d9479ff0cbd2c42be30e4278b282 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 10:06:36 -0700 Subject: [PATCH 1/8] Build EvidenceBench v4 evaluation suite --- .github/workflows/test.yml | 8 + BENCHMARK_CARD_V4.md | 67 +++ HARVEY_LAB_COMPARISON.md | 86 ++++ METHODOLOGY_V4.md | 128 +++++ PRIVATE_REPOSITORY_V4.md | 34 ++ README.md | 60 ++- V4_RELEASE_GATES.md | 48 ++ data/fre-2025-rules.json | 2 +- data/v4/dev/doctrine.jsonl | 8 + .../email-authentication/documents/record.txt | 12 + .../dev/matter/email-authentication/task.json | 71 +++ .../expert-gatekeeping/documents/record.txt | 12 + .../dev/matter/expert-gatekeeping/task.json | 71 +++ models/v4/openrouter-doctrine.example.json | 12 + models/v4/openrouter-matter.example.json | 13 + pyproject.toml | 4 +- schemas/v4/doctrine-item.schema.json | 125 +++++ schemas/v4/matter-task.schema.json | 111 +++++ src/evidencebench/cli.py | 178 +++++++ src/evidencebench/datasets_v4.py | 37 ++ src/evidencebench/models_v4.py | 329 ++++++++++++ src/evidencebench/release_v4.py | 94 ++++ src/evidencebench/runner_v4.py | 469 ++++++++++++++++++ src/evidencebench/scoring_v4.py | 359 ++++++++++++++ src/evidencebench/statistics_v4.py | 174 +++++++ src/evidencebench/validation_v4.py | 233 +++++++++ tests/test_v4.py | 170 +++++++ uv.lock | 2 +- 28 files changed, 2905 insertions(+), 12 deletions(-) create mode 100644 BENCHMARK_CARD_V4.md create mode 100644 HARVEY_LAB_COMPARISON.md create mode 100644 METHODOLOGY_V4.md create mode 100644 PRIVATE_REPOSITORY_V4.md create mode 100644 V4_RELEASE_GATES.md create mode 100644 data/v4/dev/doctrine.jsonl create mode 100644 data/v4/dev/matter/email-authentication/documents/record.txt create mode 100644 data/v4/dev/matter/email-authentication/task.json create mode 100644 data/v4/dev/matter/expert-gatekeeping/documents/record.txt create mode 100644 data/v4/dev/matter/expert-gatekeeping/task.json create mode 100644 models/v4/openrouter-doctrine.example.json create mode 100644 models/v4/openrouter-matter.example.json create mode 100644 schemas/v4/doctrine-item.schema.json create mode 100644 schemas/v4/matter-task.schema.json create mode 100644 src/evidencebench/datasets_v4.py create mode 100644 src/evidencebench/models_v4.py create mode 100644 src/evidencebench/release_v4.py create mode 100644 src/evidencebench/runner_v4.py create mode 100644 src/evidencebench/scoring_v4.py create mode 100644 src/evidencebench/statistics_v4.py create mode 100644 src/evidencebench/validation_v4.py create mode 100644 tests/test_v4.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45932d1..fff553e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,3 +15,11 @@ jobs: python-version: '3.12' - run: uv run --python 3.12 python -m unittest discover -s tests -v - run: uv run --python 3.12 evidencebench validate data/dev-v1.jsonl + - run: uv run --python 3.12 evidencebench validate-v4 --track doctrine --input data/v4/dev/doctrine.jsonl + - run: uv run --python 3.12 evidencebench validate-v4 --track matter --input data/v4/dev/matter + - name: Confirm draft fixtures cannot be published as official + run: | + if uv run --python 3.12 evidencebench validate-v4 --track doctrine --input data/v4/dev/doctrine.jsonl --official; then + echo "Draft v4 data incorrectly passed the official gate" + exit 1 + fi diff --git a/BENCHMARK_CARD_V4.md b/BENCHMARK_CARD_V4.md new file mode 100644 index 0000000..4bcc9d8 --- /dev/null +++ b/BENCHMARK_CARD_V4.md @@ -0,0 +1,67 @@ +# EvidenceBench v4 benchmark card + +## Intended use + +EvidenceBench v4 is a research evaluation for evidence-law reasoning and +closed-universe legal-agent execution. Appropriate uses include model +comparison, harness ablation, retrieval/tool experiments, error analysis, and +targeted post-training research. + +It is not legal advice, a bar examination, a general legal-intelligence score, +or evidence that a system can work without attorney review. + +## Unit of evaluation + +Doctrine units are independently authored fact patterns. Matter units are +closed-universe workspaces containing instructions, documents, expected +deliverables, and atomic criteria. `family_id` is the resampling and leakage +boundary. + +## Public data + +`data/v4/dev` contains eight Doctrine items and two Matter tasks. All are +synthetic and marked `DRAFT`. They exercise schemas, runners, validators, and +scorers. They must not be described as the v4 holdout or used for an official +leaderboard. + +## Private data + +The official set will remain under Objection Academy control in a private +source-controlled repository. A public release commits to it by cryptographic +hash and reports counts and coverage without releasing answers or item-level +outputs. + +## Primary metric + +The single overall score is the equal-weighted mean of Doctrine and Matter. +Both track scores and all submetrics remain mandatory, because equal headline +scores can conceal materially different capabilities. + +## Quality controls + +- independent legal review is required for every official item; +- authors cannot serve as their own sole reviewer; +- exact IDs, document hashes, citation existence, and rubric consistency are + validated automatically; +- DRAFT data fails the official validation command; +- public/holdout family and exact-stem overlap is rejected; +- headline scoring is deterministic; +- uncertainty is clustered by task family; and +- official results require repeated runs and disclosure of failures and costs. + +## Foreseeable risks + +Contamination can inflate results. Synthetic matters may not capture the +ambiguity of real litigation records. The pinned authority corpus can become +stale. Rubric incompleteness can penalize a legally sound alternative. +Provider routing can introduce unobserved model variation. Volunteer review can +create selection bias and inconsistent depth. + +Mitigations include a sealed family-level holdout, annual authority freezes, +adjudicated challenge logs, reviewer calibration, route disclosure, and +prospective versioning rather than silent answer changes. + +## Maintainer + +Objection Academy. Security or embargo concerns should be reported privately to +dylan@examp.com. diff --git a/HARVEY_LAB_COMPARISON.md b/HARVEY_LAB_COMPARISON.md new file mode 100644 index 0000000..9e34ddb --- /dev/null +++ b/HARVEY_LAB_COMPARISON.md @@ -0,0 +1,86 @@ +# Harvey LAB and EvidenceBench v4 + +This design comparison was frozen on July 25, 2026. Harvey LAB is evolving, so +counts are descriptive rather than permanent. + +| Dimension | Harvey LAB | EvidenceBench v4 | +|---|---|---| +| Primary target | Long-horizon legal work across many practice areas | Evidence-law doctrine and litigation-record analysis | +| Unit | Matter files, instruction, and reviewable work product | Doctrine item or closed-universe evidence matter | +| Breadth | Current repository documentation reports 1,660 tasks and roughly 101,000 criteria | v4 candidate has 8 public Doctrine and 2 public Matter development fixtures; official set is not yet authored | +| Tools | File read/edit/write, glob, grep, bash, and document-format skills | List/read/search documents and write outputs; no model shell or network | +| Core grading | Semantic LLM judge for each binary criterion; task score uses all-pass | Deterministic structured criteria; weighted track score plus all-critical Matter resolution | +| Citation treatment | A rubric can require citations | Citations are normalized against a pinned authority corpus; invalid, nonexistent, and unsupported authorities are separated | +| Uncertainty | Results emphasize all-pass and criterion pass rates | Family-clustered 95% bootstrap intervals, repeated-run requirement, and weight sensitivity | +| Contamination strategy | Public tasks plus a held-out set | Public DRAFT development set plus a private family-separated holdout with public cryptographic commitments | + +## Lessons adopted + +Matter-centric assignments are a better frontier-model test than more +multiple-choice variants. v4 therefore requires agents to find facts across +files and create artifacts. Criteria are atomic, attached to material legal, +authority, factual, and delivery requirements, and critical omissions produce +a strict task-resolution failure. The harness records transcripts and actual +deliverables. + +LAB also makes a strong case for treating the harness as part of the evaluated +system. EvidenceBench records prompt versions, tool policy, route, token and +turn limits, failures, and costs rather than labeling a result with only a +model name. + +## Deliberate differences + +EvidenceBench retains a Doctrine track because controlled diagnostics can +localize whether a failure came from the legal rule, issue spotting, authority +selection, factual grounding, or agent execution. A Matter-only score cannot +separate these causes cleanly. + +The official v4 score does not depend on an LLM judge. LAB's semantic judge is +flexible for diverse prose work product, but it adds judge-model sensitivity, +cost, and a second failure surface. EvidenceBench narrows its outputs enough to +score headline criteria deterministically. Human and LLM qualitative review +may be published separately. + +EvidenceBench also measures citation reliability as a first-class construct. +An unparseable citation remains in the precision denominator, and the evaluator +distinguishes a fabricated authority from a real authority omitted from the +annotation. + +## Improvements over EvidenceBench v3 + +v3 was useful for closed-book answer and citation testing but was too narrow +for frontier agent research. Its multiple-choice format encouraged saturation; +its public development and mechanically varied holdout families created +leakage risk; an explanation was requested but not scored; invalid citations +could be normalized away; and aggregate tables lacked uncertainty. + +v4 addresses those problems with: + +- independent family IDs and a public/holdout overlap audit; +- a private, source-controlled holdout; +- structured issue and fact grounding; +- a long-horizon Matter track; +- explicit DRAFT/APPROVED review state enforced by the official validator; +- retry-on-transport-only execution; +- deterministic scoring and citation error accounting; +- one preregistered overall score with mandatory track disclosure; and +- clustered confidence intervals, run replication, and weighting sensitivity. + +## What remains before a defensible launch + +The v4 software is not the v4 benchmark corpus. The official sample needs a +coverage matrix, original families, independent professional review, +adjudication, pilot-based item analysis, and a preregistered release plan. +Only after those gates pass should Objection Academy run paid models or publish +a leaderboard. + +## Sources + +- Harvey announcement: + +- Harvey LAB repository: + +- Harvey evaluation methodology: + +- Vals AI independent benchmark page: + diff --git a/METHODOLOGY_V4.md b/METHODOLOGY_V4.md new file mode 100644 index 0000000..04f9b76 --- /dev/null +++ b/METHODOLOGY_V4.md @@ -0,0 +1,128 @@ +# EvidenceBench v4 methodology + +Status: implementation candidate. The public development fixtures are `DRAFT`, +not an official benchmark release. + +## Research question + +EvidenceBench v4 measures whether an LLM system can reach sound evidence-law +outcomes, identify the controlling issues, cite authorities that exist in a +pinned corpus, connect conclusions to supplied facts, and complete a +record-grounded work product. It does not measure whether a system can replace +a lawyer or safely practice law without supervision. + +## Why two tracks + +Harvey's Legal Agent Benchmark demonstrates the value of matter-centric, +long-horizon assignments: an instruction, a closed-universe client matter, +filesystem tools, reviewable deliverables, and atomic criteria. Its first +release described more than 1,200 tasks across 24 practice areas and more than +75,000 expert-written criteria. EvidenceBench adopts the parts that fit +evidence-law research while retaining a controlled doctrine track. + +The Doctrine track is a short-horizon diagnostic. It tests rulings, issue +recognition, authority selection, fact grounding, and confidence without tools. +The Matter track is a constrained agent task. It tests document discovery, +record synthesis, evidence analysis, citation, and delivery through read, +literal-search, and output-write tools. + +The two tracks answer different questions and are always reported separately. +One headline score is also reported: + +`EvidenceBench v4 = 0.50 × Doctrine + 0.50 × Matter` + +The fixed equal weighting prevents the larger track from silently dominating. +Every release also reports 40/60 and 60/40 sensitivity values. A submission +must complete both tracks; a missing track does not receive a headline score. + +## Doctrine scoring + +Each item receives: + +- outcome accuracy: 40%; +- issue-code F1: 25%; +- authority F1: 20%; +- issue-to-fact grounding F1: 10%; and +- Brier-derived calibration (`1 - (confidence - correctness)^2`): 5%. + +Authority precision counts every nonempty submitted string in its denominator. +Unparseable citations are not discarded. Authority recall is the fraction of +required authority groups for which at least one accepted citation is supplied. +The benchmark separately records invalid, hallucinated, and real-but-unsupported +authorities. + +## Matter scoring + +Expert rubrics are atomic and binary. Scoring dimensions are: + +- legal conclusions: 50%; +- authority grounding: 20%; +- factual/record grounding: 15%; and +- deliverable completeness: 15%. + +The Matter score is the weighted mean of dimension-level pass rates. In +addition, a task-resolution flag passes only if every critical criterion +passes. This preserves the useful strictness of all-pass review without +collapsing the headline metric into a mostly-zero statistic. Criteria marked +`review_only` can support qualitative error analysis but cannot change the +official score. + +The evaluator derives deliverable existence from the output filesystem; a +model cannot earn credit merely by claiming it wrote a file. + +## Determinism and judges + +All headline scoring is deterministic. No LLM judge participates in the +official score. A later research release may publish blinded human or LLM +quality ratings as secondary fields, but those ratings must report the judge +prompt, model, agreement, and adjudication protocol and may not be merged into +the v4 score. + +Model sampling is temperature zero with a recorded seed when the route supports +it. Each official model result must include at least three independent runs. +The leaderboard reports the run mean, run dispersion, and family-clustered 95% +bootstrap confidence intervals. + +## Dataset design + +Public development families are for integration and evaluator testing. +Official prompts, annotations, matter files, and item-level responses remain in +a private, access-controlled repository. Public and holdout data may not share +a `family_id`. Exact normalized stem overlap is automatically rejected; release +review must additionally test semantic overlap and contamination. + +Synthetic matters must use fictional parties and facts. Real cases may be used +only when the source, license, holding, current-law status, and retrieval hash +are recorded. Every official item must name an author and at least one +independent reviewer and must have status `APPROVED`. + +## Required reporting + +An official result publishes: + +- benchmark and corpus versions, dataset commitments, and prompt versions; +- exact model slug and OpenRouter route metadata available for the run; +- parameters, tool policy, maximum turns, token usage, cost, failures, and + fallback behavior; +- track scores, submetrics, task-resolution rate, invalid and hallucinated + citation counts; +- family-clustered confidence intervals and weighting sensitivity; and +- all deviations from the reference harness. + +## Known limitations + +EvidenceBench focuses on United States federal evidence doctrine. Development +fixtures are small and synthetic. Deterministic issue codes and rubrics reduce +judge variance but can under-credit unanticipated sound analysis; challenges +must be adjudicated and incorporated prospectively in a versioned annotation. +OpenRouter can route the same model slug through different upstream providers, +so an official run must pin and disclose route behavior where the service +permits it. + +## References + +- Harvey, “Open-Sourcing Harvey's Long Horizon Legal Agent Benchmark,” + +- Harvey LAB source, +- Vals AI LAB results and harness description, + diff --git a/PRIVATE_REPOSITORY_V4.md b/PRIVATE_REPOSITORY_V4.md new file mode 100644 index 0000000..355d150 --- /dev/null +++ b/PRIVATE_REPOSITORY_V4.md @@ -0,0 +1,34 @@ +# Private v4 repository contract + +The official holdout should live in a dedicated private repository controlled +by Objection Academy, rather than in the public website repository. + +Recommended layout: + +```text +EvidenceBench-Private/ + candidates/doctrine/ + candidates/matter/ + holdout/v4/doctrine.jsonl + holdout/v4/matter/ + reviews/ + adjudications/ + runs/raw/ + releases/ +``` + +Only reviewed data moves from `candidates` to `holdout`. Branch protection +should require an authorized review for holdout, review, adjudication, and +release paths. Access should be limited to maintainers and reviewers who need +the sealed materials. The website should receive only signed or hashed release +metadata and aggregate results. + +The private repository must not contain API keys, provider credentials, or +unredacted confidential client material. Use synthetic or properly licensed +records. Item-level model outputs and transcripts remain private because they +can reveal holdout content. + +Every public release should include commitments for the Doctrine data, each +Matter task manifest and document inventory, scoring code, prompt templates, +and run protocol. A verifier with private access must be able to reproduce those +hashes from a clean checkout. diff --git a/README.md b/README.md index 358a775..1b30692 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,32 @@ # EvidenceBench -EvidenceBench is Objection Academy's public benchmark for evidence-law answer -accuracy and legal-authority citation reliability. It is an -educational research project, not legal advice and not a substitute for -professional legal research. +EvidenceBench is Objection Academy's evidence-law benchmark. v4 adds a +closed-book Doctrine track and a closed-universe Matter agent track while +preserving the v3 evaluator for reproducibility. It is an educational research +project, not legal advice or a substitute for professional legal research. + +## v4 status + +The v4 evaluator, schemas, OpenRouter runners, scoring, integrity checks, and +public development fixtures are implemented. The fixtures are deliberately +marked `DRAFT`; they validate for development and fail the `--official` gate. +EvidenceBench v4 must not be presented as an official leaderboard until the +legal-review and release gates in [V4_RELEASE_GATES.md](V4_RELEASE_GATES.md) +are complete. + +v4 reports a single overall score: + +`0.50 × Doctrine score + 0.50 × Matter score` + +It also reports both track scores, submetrics, strict Matter task-resolution +rate, family-clustered 95% bootstrap intervals, and 40/60 versus 60/40 weight +sensitivity. Headline scoring is deterministic; LLM judges cannot change the +official score. + +See [METHODOLOGY_V4.md](METHODOLOGY_V4.md) and +[BENCHMARK_CARD_V4.md](BENCHMARK_CARD_V4.md). The design rationale and the +specific lessons taken from Harvey LAB are in +[HARVEY_LAB_COMPARISON.md](HARVEY_LAB_COMPARISON.md). ## What is public @@ -29,7 +52,28 @@ annotations, and item-level results are not public. Once approved, published manifests will commit to the holdout with a SHA-256 hash, counts, categories, protocol hash, and aggregate metrics. -## Commands +## v4 commands + +```bash +uv run --python 3.12 evidencebench validate-v4 --track doctrine --input data/v4/dev/doctrine.jsonl +uv run --python 3.12 evidencebench validate-v4 --track matter --input data/v4/dev/matter +uv run --python 3.12 evidencebench score-v4 --track doctrine --input data/v4/dev/doctrine.jsonl --responses results/v4/doctrine-responses.jsonl --output results/v4/doctrine-scores.json +uv run --python 3.12 evidencebench score-v4 --track matter --input data/v4/dev/matter --responses results/v4/matter-responses.jsonl --output results/v4/matter-scores.json +uv run --python 3.12 evidencebench summarize-v4 --doctrine-scores results/v4/doctrine-scores.json --matter-scores results/v4/matter-scores.json --output results/v4/summary.json +uv run --python 3.12 evidencebench manifest-v4 --doctrine data/v4/dev/doctrine.jsonl --matter data/v4/dev/matter --output results/v4/development-manifest.json +``` + +The runners use OpenRouter and read `OPENROUTER_API_KEY` from the process +environment. No key belongs in a tracked manifest. Doctrine runs have tools +disabled. Matter runs expose only document listing, document reading, literal +search, and output writing; the model receives no shell or network tool. + +```bash +uv run --python 3.12 evidencebench run-v4-doctrine --manifest models/v4/openrouter-doctrine.example.json --items data/v4/dev/doctrine.jsonl --output results/v4/doctrine-responses.jsonl +uv run --python 3.12 evidencebench run-v4-matter --manifest models/v4/openrouter-matter.example.json --tasks data/v4/dev/matter --output results/v4/matter-run +``` + +## v3 commands ```bash uv run --python 3.12 evidencebench validate data/dev-v1.jsonl @@ -37,10 +81,10 @@ uv run --python 3.12 evidencebench score --questions data/dev-v1.jsonl --respons uv run --python 3.12 evidencebench export-web --manifest results/v1.0.0/manifest.json --output exports/latest.json ``` -`run` is deliberately opt-in. It requires a model manifest plus the provider +The v3 `run` command is deliberately opt-in. It requires a model manifest plus the provider credentials named in that manifest. It never enables web search or tools. -## Metrics +## v3 metrics Each item receives answer accuracy, authority precision, authority recall, citation F1, citation-existence status, and unsupported-citation status. @@ -54,7 +98,7 @@ published LEXIS citation. v3 uses the Federal Rules of Evidence in effect December 1, 2025. The pending Rule 801 amendment effective December 1, 2026 is excluded. -## Official-run gate +## v3 official-run gate The sealed holdout has an attorney-approved citation and content review record and is committed in each public manifest by SHA-256. The normalized government diff --git a/V4_RELEASE_GATES.md b/V4_RELEASE_GATES.md new file mode 100644 index 0000000..cc0f282 --- /dev/null +++ b/V4_RELEASE_GATES.md @@ -0,0 +1,48 @@ +# EvidenceBench v4 release gates + +No result is “official EvidenceBench v4” until all gates pass. + +## Data gate + +- [ ] Target sample and coverage plan are frozen before model runs. +- [ ] Every item/task is `APPROVED` by an author and an independent reviewer. +- [ ] High-risk or disputed annotations have adjudication records. +- [ ] Citation corpus and effective dates are verified. +- [ ] Matter document hashes match. +- [ ] Public/holdout family, exact-text, and semantic leakage audits pass. +- [ ] No benchmark item is a deterministic transformation of a public item. + +## Measurement gate + +- [ ] Gold-response tests cover every scoring branch. +- [ ] Adversarial tests cover malformed JSON, path traversal, missing outputs, + invalid citations, fabricated citations, refusals, and partial failures. +- [ ] A frozen scoring version and prompt version are recorded. +- [ ] Family-clustered confidence intervals are reported. +- [ ] At least three runs per system are completed. +- [ ] Track-weight sensitivity is reported. +- [ ] Human agreement and adjudication statistics are published. + +## Execution gate + +- [ ] Exact model IDs and OpenRouter route/fallback policy are recorded. +- [ ] Matter tools cannot read outside documents or write outside outputs. +- [ ] Network and shell access are unavailable to the model. +- [ ] Time, token, turn, and cost limits are identical across compared systems. +- [ ] Failures remain failures; only retryable transport errors are retried. +- [ ] Transcripts and item-level outputs remain sealed for audit. + +## Publication gate + +- [ ] Benchmark card and methodology match the executable code. +- [ ] Dataset and protocol SHA-256 commitments are public. +- [ ] A preregistered analysis plan identifies the primary comparison. +- [ ] No ranking claim is made when confidence intervals and repeated-run + variation do not support it. +- [ ] Aggregate results include failure, invalid-citation, and + hallucinated-citation counts. +- [ ] The release clearly distinguishes public development data, private + holdout data, and any unreviewed candidate pool. + +Passing the software tests alone is insufficient. Until the legal-review and +measurement gates pass, the implementation is a v4 candidate. diff --git a/data/fre-2025-rules.json b/data/fre-2025-rules.json index 7ef59b8..6e5f5fd 100644 --- a/data/fre-2025-rules.json +++ b/data/fre-2025-rules.json @@ -4,7 +4,7 @@ "rules": { "101": [""], "102": [""], "103": ["", "(a)", "(b)", "(c)", "(d)"], "104": ["", "(a)", "(b)", "(c)", "(d)", "(e)"], "105": [""], "106": [""], "107": [""], "201": ["", "(a)", "(b)", "(c)", "(d)", "(e)", "(f)"], "301": [""], "302": [""], - "401": [""], "402": [""], "403": [""], "404": ["", "(a)", "(a)(1)", "(a)(2)", "(b)", "(b)(1)", "(b)(2)"], "405": ["", "(a)", "(b)"], "406": [""], "407": [""], "408": [""], "409": [""], "410": [""], "411": [""], "412": [""], "413": [""], "414": [""], "415": [""], + "401": [""], "402": [""], "403": [""], "404": ["", "(a)", "(a)(1)", "(a)(2)", "(b)", "(b)(1)", "(b)(2)", "(b)(3)"], "405": ["", "(a)", "(b)"], "406": [""], "407": [""], "408": [""], "409": [""], "410": [""], "411": [""], "412": [""], "413": [""], "414": [""], "415": [""], "501": [""], "502": ["", "(a)", "(b)", "(c)", "(d)", "(e)", "(f)", "(g)"], "601": [""], "602": [""], "603": [""], "604": [""], "605": [""], "606": ["", "(a)", "(b)"], "607": [""], "608": ["", "(a)", "(b)"], "609": ["", "(a)", "(a)(1)", "(a)(2)", "(b)"], "610": [""], "611": ["", "(a)", "(b)", "(c)"], "612": [""], "613": ["", "(a)", "(b)"], "614": ["", "(a)", "(b)", "(c)"], "615": [""], "616": [""], "701": ["", "(a)", "(b)", "(c)"], "702": ["", "(a)", "(b)", "(c)", "(d)"], "703": [""], "704": ["", "(a)", "(b)"], "705": [""], "706": [""], "707": [""], diff --git a/data/v4/dev/doctrine.jsonl b/data/v4/dev/doctrine.jsonl new file mode 100644 index 0000000..adb6a72 --- /dev/null +++ b/data/v4/dev/doctrine.jsonl @@ -0,0 +1,8 @@ +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-001","family_id":"V4F-104A-001","category":"judicial-role","difficulty":"medium","jurisdiction":"federal","stem":"At a jury trial, the proponent offers a statement under the coconspirator provision. Whether a conspiracy existed and whether the statement furthered it are disputed. Who decides those preliminary facts, and under what standard?","facts":[{"id":"F1","text":"The proponent invokes the coconspirator exclusion from hearsay."},{"id":"F2","text":"The existence and scope of the alleged conspiracy are disputed."}],"allowed_rulings":["admit","exclude","defer"],"gold":{"ruling":"defer","issue_codes":["FRE_104A_PRELIMINARY_FACT"],"required_authority_groups":[["FRE 104(a)"]],"accepted_authorities":["FRE 104(a)"],"grounding":[{"issue_code":"FRE_104A_PRELIMINARY_FACT","fact_ids":["F1","F2"]}],"rationale":"The court decides admissibility predicates under Rule 104(a); the development item uses defer because the facts supplied do not establish whether the predicate is met."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"allocate judge-jury responsibility","source":"synthetic"}} +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-002","family_id":"V4F-106-001","category":"completeness","difficulty":"medium","jurisdiction":"federal","stem":"A party introduces selected sentences from a witness's recorded interview. The opponent asks to introduce nearby portions at the same time because the excerpts otherwise imply the witness made an unqualified identification. The nearby portions state that the witness was uncertain. How should the court rule?","facts":[{"id":"F1","text":"The first party offered only selected sentences from a recorded interview."},{"id":"F2","text":"Nearby portions qualify the witness's identification as uncertain."},{"id":"F3","text":"Without the qualifying portions, the selected excerpt implies an unqualified identification."}],"allowed_rulings":["admit","exclude","limit"],"gold":{"ruling":"admit","issue_codes":["FRE_106_COMPLETENESS"],"required_authority_groups":[["FRE 106"]],"accepted_authorities":["FRE 106"],"grounding":[{"issue_code":"FRE_106_COMPLETENESS","fact_ids":["F1","F2","F3"]}],"rationale":"Rule 106 permits an adverse party to require contemporaneous introduction of another part that in fairness ought to be considered at the same time."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"apply completeness","source":"synthetic"}} +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-003","family_id":"V4F-404B-001","category":"character-and-other-acts","difficulty":"hard","jurisdiction":"federal","stem":"In a fraud prosecution, the government offers evidence that the defendant used the same unusual three-step invoicing method in a prior transaction. It offers the evidence to prove identity, gives reasonable written pretrial notice, and the court finds identity genuinely disputed. The evidence is not offered to prove criminal propensity. Subject to Rule 403, how should the court rule?","facts":[{"id":"F1","text":"Identity is genuinely disputed."},{"id":"F2","text":"The prior transaction used the same unusual three-step method as the charged transaction."},{"id":"F3","text":"The government identified identity as the non-propensity purpose and gave reasonable written pretrial notice."},{"id":"F4","text":"The evidence is not offered to prove propensity."}],"allowed_rulings":["admit","exclude","limit"],"gold":{"ruling":"admit","issue_codes":["FRE_404B_NONPROPENSITY"],"required_authority_groups":[["FRE 404(b)(2)"],["FRE 404(b)(3)"]],"accepted_authorities":["FRE 404(b)(2)","FRE 404(b)(3)"],"grounding":[{"issue_code":"FRE_404B_NONPROPENSITY","fact_ids":["F1","F2","F3","F4"]}],"rationale":"The facts specify a permitted identity purpose and compliance with the criminal-case notice provision; Rule 403 remains separately applicable."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"distinguish propensity from permitted purpose","source":"synthetic"}} +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-004","family_id":"V4F-602-001","category":"witness-competency","difficulty":"easy","jurisdiction":"federal","stem":"A witness proposes to testify that a traffic light was red. On voir dire, the witness admits being inside a windowless building and learning the light's color only from a bystander afterward. The testimony is offered for its truth. How should the court rule?","facts":[{"id":"F1","text":"The witness was inside a windowless building when the event occurred."},{"id":"F2","text":"The witness learned the light's color only from a bystander."},{"id":"F3","text":"The witness did not personally perceive the traffic light."}],"allowed_rulings":["admit","exclude","limit"],"gold":{"ruling":"exclude","issue_codes":["FRE_602_PERSONAL_KNOWLEDGE"],"required_authority_groups":[["FRE 602"]],"accepted_authorities":["FRE 602"],"grounding":[{"issue_code":"FRE_602_PERSONAL_KNOWLEDGE","fact_ids":["F1","F2","F3"]}],"rationale":"The record supplies no evidence sufficient to support a finding that the witness has personal knowledge."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"identify lack of personal knowledge","source":"synthetic"}} +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-005","family_id":"V4F-608B-001","category":"impeachment","difficulty":"medium","jurisdiction":"federal","stem":"On cross-examination, counsel asks a witness about a prior dishonest act that did not result in a conviction. The court finds the act probative of the witness's character for truthfulness. After the witness denies it, counsel seeks to introduce a separate document solely to prove that act. How should the court treat the document?","facts":[{"id":"F1","text":"The prior act did not result in a conviction."},{"id":"F2","text":"The act is probative of the witness's character for truthfulness."},{"id":"F3","text":"The separate document is offered solely to prove the specific act after the witness denied it."}],"allowed_rulings":["admit","exclude","limit"],"gold":{"ruling":"exclude","issue_codes":["FRE_608B_EXTRINSIC_EVIDENCE"],"required_authority_groups":[["FRE 608(b)"]],"accepted_authorities":["FRE 608(b)"],"grounding":[{"issue_code":"FRE_608B_EXTRINSIC_EVIDENCE","fact_ids":["F1","F2","F3"]}],"rationale":"Rule 608(b) permits qualifying inquiry on cross-examination in the court's discretion but bars extrinsic evidence to prove the specific instance for that character purpose."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"separate inquiry from extrinsic proof","source":"synthetic"}} +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-006","family_id":"V4F-702-001","category":"expert-testimony","difficulty":"hard","jurisdiction":"federal","stem":"An engineer is qualified and offers relevant accident-reconstruction testimony. The proponent shows that the method is generally reliable, but the engineer ignored two measurements in this case and cannot explain why the method was applied differently from its protocol. The proponent does not establish reliable application to the case facts. How should the court rule on this opinion?","facts":[{"id":"F1","text":"The engineer is qualified and the subject would help the factfinder."},{"id":"F2","text":"The underlying method is generally reliable."},{"id":"F3","text":"The engineer ignored two case measurements and departed from the method's protocol without explanation."},{"id":"F4","text":"The proponent did not establish reliable application to the case facts."}],"allowed_rulings":["admit","exclude","limit"],"gold":{"ruling":"exclude","issue_codes":["FRE_702_RELIABLE_APPLICATION"],"required_authority_groups":[["FRE 702(d)"]],"accepted_authorities":["FRE 702(d)"],"grounding":[{"issue_code":"FRE_702_RELIABLE_APPLICATION","fact_ids":["F3","F4"]}],"rationale":"Rule 702 separately requires the proponent to demonstrate that the expert reliably applied the principles and methods to the case facts."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"distinguish method reliability from application","source":"synthetic"}} +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-007","family_id":"V4F-8036-001","category":"hearsay","difficulty":"medium","jurisdiction":"federal","stem":"A company offers a sales record made automatically at the time of sale as part of its regular business process. A qualified records custodian lays the required foundation. The opponent presents evidence that an employee altered this record a week later for purposes of litigation. How should the court rule?","facts":[{"id":"F1","text":"The sales record was initially made contemporaneously in the regular course."},{"id":"F2","text":"A qualified custodian supplies the ordinary foundation."},{"id":"F3","text":"Evidence shows that an employee altered this record a week later for litigation."}],"allowed_rulings":["admit","exclude","limit"],"gold":{"ruling":"exclude","issue_codes":["FRE_8036_TRUSTWORTHINESS"],"required_authority_groups":[["FRE 803(6)"]],"accepted_authorities":["FRE 803(6)"],"grounding":[{"issue_code":"FRE_8036_TRUSTWORTHINESS","fact_ids":["F1","F2","F3"]}],"rationale":"Even when the ordinary foundation is present, Rule 803(6) does not apply when the source, method, circumstances, or preparation indicate a lack of trustworthiness."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"apply trustworthiness proviso","source":"synthetic"}} +{"schema_version":"4.0","track":"doctrine","id":"V4D-DEV-008","family_id":"V4F-901B4-001","category":"authentication","difficulty":"medium","jurisdiction":"federal","stem":"A party offers an email purportedly written by an opposing manager. No witness saw it being sent. The message contains a nickname known only within the manager's small team, describes a nonpublic meeting accurately, and came from the address the manager routinely used. Is this evidence sufficient to permit an authenticity finding, leaving ultimate weight to the jury?","facts":[{"id":"F1","text":"The email contains an internal nickname known only within the manager's small team."},{"id":"F2","text":"The email accurately describes a nonpublic meeting."},{"id":"F3","text":"The email came from the address the manager routinely used."},{"id":"F4","text":"No witness observed the email being sent."}],"allowed_rulings":["admit","exclude","defer"],"gold":{"ruling":"admit","issue_codes":["FRE_901B4_DISTINCTIVE_CHARACTERISTICS"],"required_authority_groups":[["FRE 901(b)(4)"]],"accepted_authorities":["FRE 901(b)(4)"],"grounding":[{"issue_code":"FRE_901B4_DISTINCTIVE_CHARACTERISTICS","fact_ids":["F1","F2","F3"]}],"rationale":"The contents and circumstances provide a permissible distinctive-characteristics route to authentication; eyewitness testimony is not invariably required."},"corpus_version":"fre-2025","review":{"status":"DRAFT","author_id":"oa-v4-drafter","reviewer_ids":[]},"dimensions":{"skill":"authenticate using circumstantial features","source":"synthetic"}} diff --git a/data/v4/dev/matter/email-authentication/documents/record.txt b/data/v4/dev/matter/email-authentication/documents/record.txt new file mode 100644 index 0000000..855f2e2 --- /dev/null +++ b/data/v4/dev/matter/email-authentication/documents/record.txt @@ -0,0 +1,12 @@ +MATTER RECORD — DEVELOPMENT FIXTURE + +[F1] On March 3, Ana Ruiz received an email from marcus@northstar.test. +[F2] Northstar's directory lists that address for manager Marcus Lee. +[F3] The email calls the pricing project “Blue Finch,” a nickname used only by +the five-person pricing team. +[F4] It accurately says that Ana objected during a closed March 2 meeting. +[F5] The exported message has no complete header metadata or server log. +[F6] Ana did not see Marcus type or send the message. + +The proponent asks the court to admit the email. The opponent argues that only +an eyewitness or complete header metadata could authenticate it. diff --git a/data/v4/dev/matter/email-authentication/task.json b/data/v4/dev/matter/email-authentication/task.json new file mode 100644 index 0000000..0289fab --- /dev/null +++ b/data/v4/dev/matter/email-authentication/task.json @@ -0,0 +1,71 @@ +{ + "schema_version": "4.0", + "track": "matter", + "id": "V4M-DEV-001", + "family_id": "V4MF-901-001", + "title": "Email authentication assessment", + "task_type": "record_review", + "instructions": "Review the record and determine whether the proponent has supplied evidence sufficient to support a finding that the email is what the proponent claims. Address the opponent's assertion that eyewitness testimony or complete header metadata is mandatory.", + "documents": [ + { + "path": "record.txt", + "sha256": "c2d707688713dd810dcfaa93ed90ae322e938691a7546a617acfb160eb2a907c" + } + ], + "deliverables": ["findings.json"], + "criteria": [ + { + "id": "C1", + "dimension": "legal", + "title": "Reaches the supported authentication disposition", + "issue_code": "FRE_901_DISTINCTIVE_CHARACTERISTICS", + "expected_disposition": "sufficient", + "critical": true + }, + { + "id": "C2", + "dimension": "authority", + "title": "Grounds the authentication route in Rule 901", + "issue_code": "FRE_901_DISTINCTIVE_CHARACTERISTICS", + "expected_disposition": "sufficient", + "required_authority_groups": [["FRE 901(a)"], ["FRE 901(b)(4)"]], + "accepted_authorities": ["FRE 901(a)", "FRE 901(b)(4)"], + "critical": true + }, + { + "id": "C3", + "dimension": "fact", + "title": "Uses the distinctive contents and address evidence", + "issue_code": "FRE_901_DISTINCTIVE_CHARACTERISTICS", + "expected_disposition": "sufficient", + "required_fact_ids": ["F2", "F3", "F4"], + "required_record_refs": ["record.txt"], + "critical": true + }, + { + "id": "C4", + "dimension": "deliverable", + "title": "Creates the structured findings deliverable", + "deliverable": "findings.json", + "critical": true + }, + { + "id": "R1", + "dimension": "legal", + "title": "Explains the difference between sufficiency and ultimate weight", + "issue_code": "FRE_901_DISTINCTIVE_CHARACTERISTICS", + "critical": false, + "review_only": true + } + ], + "corpus_version": "fre-2025", + "review": { + "status": "DRAFT", + "author_id": "oa-v4-drafter", + "reviewer_ids": [] + }, + "dimensions": { + "skill": "record-grounded authentication analysis", + "source": "synthetic" + } +} diff --git a/data/v4/dev/matter/expert-gatekeeping/documents/record.txt b/data/v4/dev/matter/expert-gatekeeping/documents/record.txt new file mode 100644 index 0000000..7080967 --- /dev/null +++ b/data/v4/dev/matter/expert-gatekeeping/documents/record.txt @@ -0,0 +1,12 @@ +MATTER RECORD — DEVELOPMENT FIXTURE + +[F1] Dr. Chen is a licensed mechanical engineer with 18 years of crash work. +[F2] The delta-v method she identifies is accepted for accident reconstruction. +[F3] The method's written protocol requires measurements from both vehicles. +[F4] Dr. Chen measured only the claimant's vehicle. +[F5] Photographs and an available inspection report contained measurements for +the second vehicle, but her file does not mention them. +[F6] At deposition, she could not explain the departure from the protocol. +[F7] Her opinion would assist the jury if it satisfied Rule 702. + +Prepare a gatekeeping assessment of the proposed opinion. diff --git a/data/v4/dev/matter/expert-gatekeeping/task.json b/data/v4/dev/matter/expert-gatekeeping/task.json new file mode 100644 index 0000000..88170fe --- /dev/null +++ b/data/v4/dev/matter/expert-gatekeeping/task.json @@ -0,0 +1,71 @@ +{ + "schema_version": "4.0", + "track": "matter", + "id": "V4M-DEV-002", + "family_id": "V4MF-702-001", + "title": "Expert gatekeeping assessment", + "task_type": "record_review", + "instructions": "Review the record and determine whether the proposed accident-reconstruction opinion satisfies Rule 702. Distinguish the reliability of the general method from the reliability of its application in this matter.", + "documents": [ + { + "path": "record.txt", + "sha256": "284d1a0c13ff16040f6007541e38a494970c7be7bd13ea7ab028e0bfc91b7d95" + } + ], + "deliverables": ["findings.json"], + "criteria": [ + { + "id": "C1", + "dimension": "legal", + "title": "Reaches the supported gatekeeping disposition", + "issue_code": "FRE_702_RELIABLE_APPLICATION", + "expected_disposition": "exclude", + "critical": true + }, + { + "id": "C2", + "dimension": "authority", + "title": "Grounds reliable application in Rule 702(d)", + "issue_code": "FRE_702_RELIABLE_APPLICATION", + "expected_disposition": "exclude", + "required_authority_groups": [["FRE 702(d)"]], + "accepted_authorities": ["FRE 702(d)"], + "critical": true + }, + { + "id": "C3", + "dimension": "fact", + "title": "Identifies the unexplained protocol departure", + "issue_code": "FRE_702_RELIABLE_APPLICATION", + "expected_disposition": "exclude", + "required_fact_ids": ["F3", "F4", "F5", "F6"], + "required_record_refs": ["record.txt"], + "critical": true + }, + { + "id": "C4", + "dimension": "deliverable", + "title": "Creates the structured findings deliverable", + "deliverable": "findings.json", + "critical": true + }, + { + "id": "R1", + "dimension": "legal", + "title": "Separately discusses qualifications, helpfulness, method, and application", + "issue_code": "FRE_702_RELIABLE_APPLICATION", + "critical": false, + "review_only": true + } + ], + "corpus_version": "fre-2025", + "review": { + "status": "DRAFT", + "author_id": "oa-v4-drafter", + "reviewer_ids": [] + }, + "dimensions": { + "skill": "record-grounded expert gatekeeping", + "source": "synthetic" + } +} diff --git a/models/v4/openrouter-doctrine.example.json b/models/v4/openrouter-doctrine.example.json new file mode 100644 index 0000000..c631592 --- /dev/null +++ b/models/v4/openrouter-doctrine.example.json @@ -0,0 +1,12 @@ +{ + "provider": "openrouter", + "model_id": "replace/with-pinned-model-slug", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "max_output_tokens": 1200, + "timeout_seconds": 180, + "tools_enabled": false +} diff --git a/models/v4/openrouter-matter.example.json b/models/v4/openrouter-matter.example.json new file mode 100644 index 0000000..135c2e2 --- /dev/null +++ b/models/v4/openrouter-matter.example.json @@ -0,0 +1,13 @@ +{ + "provider": "openrouter", + "model_id": "replace/with-pinned-model-slug", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "max_output_tokens": 2000, + "max_turns": 20, + "timeout_seconds": 180, + "tools_enabled": true +} diff --git a/pyproject.toml b/pyproject.toml index cd233b4..ba68036 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "evidencebench" -version = "3.0.0" -description = "Reproducible evidence-law and Federal Rules of Evidence benchmark." +version = "4.0.0" +description = "Reproducible evidence-law benchmark for doctrine and matter-based legal agents." readme = "README.md" requires-python = ">=3.12" license = { text = "MIT" } diff --git a/schemas/v4/doctrine-item.schema.json b/schemas/v4/doctrine-item.schema.json new file mode 100644 index 0000000..6b5b5df --- /dev/null +++ b/schemas/v4/doctrine-item.schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://objectionacademy.com/evidencebench/v4/doctrine-item.schema.json", + "title": "EvidenceBench v4 Doctrine item", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "track", + "id", + "family_id", + "category", + "difficulty", + "jurisdiction", + "stem", + "facts", + "allowed_rulings", + "gold", + "corpus_version", + "review" + ], + "properties": { + "schema_version": {"const": "4.0"}, + "track": {"const": "doctrine"}, + "id": {"type": "string", "minLength": 1}, + "family_id": {"type": "string", "minLength": 1}, + "category": {"type": "string", "minLength": 1}, + "difficulty": {"enum": ["easy", "medium", "hard"]}, + "jurisdiction": {"type": "string", "minLength": 1}, + "stem": {"type": "string", "minLength": 1}, + "facts": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "text"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "text": {"type": "string", "minLength": 1} + } + } + }, + "allowed_rulings": { + "type": "array", + "minItems": 2, + "uniqueItems": true, + "items": {"enum": ["admit", "exclude", "limit", "defer"]} + }, + "gold": { + "type": "object", + "additionalProperties": false, + "required": [ + "ruling", + "issue_codes", + "required_authority_groups", + "accepted_authorities", + "grounding", + "rationale" + ], + "properties": { + "ruling": {"enum": ["admit", "exclude", "limit", "defer"]}, + "issue_codes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "required_authority_groups": { + "type": "array", + "items": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + } + }, + "accepted_authorities": { + "type": "array", + "items": {"type": "string", "minLength": 1} + }, + "grounding": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["issue_code", "fact_ids"], + "properties": { + "issue_code": {"type": "string", "minLength": 1}, + "fact_ids": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + } + } + } + }, + "rationale": {"type": "string", "minLength": 1} + } + }, + "corpus_version": {"type": "string", "minLength": 1}, + "review": {"$ref": "#/$defs/review"}, + "dimensions": { + "type": "object", + "additionalProperties": {"type": "string"} + } + }, + "$defs": { + "review": { + "type": "object", + "additionalProperties": false, + "required": ["status"], + "properties": { + "status": {"enum": ["DRAFT", "IN_REVIEW", "APPROVED"]}, + "author_id": {"type": ["string", "null"]}, + "reviewer_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string"} + }, + "adjudicator_id": {"type": ["string", "null"]}, + "reviewed_at": {"type": ["string", "null"], "format": "date-time"} + } + } + } +} diff --git a/schemas/v4/matter-task.schema.json b/schemas/v4/matter-task.schema.json new file mode 100644 index 0000000..b107003 --- /dev/null +++ b/schemas/v4/matter-task.schema.json @@ -0,0 +1,111 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://objectionacademy.com/evidencebench/v4/matter-task.schema.json", + "title": "EvidenceBench v4 Matter task", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "track", + "id", + "family_id", + "title", + "task_type", + "instructions", + "documents", + "deliverables", + "criteria", + "corpus_version", + "review" + ], + "properties": { + "schema_version": {"const": "4.0"}, + "track": {"const": "matter"}, + "id": {"type": "string", "minLength": 1}, + "family_id": {"type": "string", "minLength": 1}, + "title": {"type": "string", "minLength": 1}, + "task_type": {"type": "string", "minLength": 1}, + "instructions": {"type": "string", "minLength": 1}, + "documents": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + } + } + }, + "deliverables": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "criteria": { + "type": "array", + "minItems": 4, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "dimension", "title"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "dimension": { + "enum": ["legal", "authority", "fact", "deliverable"] + }, + "title": {"type": "string", "minLength": 1}, + "issue_code": {"type": ["string", "null"]}, + "expected_disposition": {"type": ["string", "null"]}, + "required_fact_ids": { + "type": "array", + "items": {"type": "string"} + }, + "required_record_refs": { + "type": "array", + "items": {"type": "string"} + }, + "required_authority_groups": { + "type": "array", + "items": { + "type": "array", + "minItems": 1, + "items": {"type": "string"} + } + }, + "accepted_authorities": { + "type": "array", + "items": {"type": "string"} + }, + "deliverable": {"type": ["string", "null"]}, + "critical": {"type": "boolean"}, + "review_only": {"type": "boolean"} + } + } + }, + "corpus_version": {"type": "string", "minLength": 1}, + "review": { + "type": "object", + "additionalProperties": false, + "required": ["status"], + "properties": { + "status": {"enum": ["DRAFT", "IN_REVIEW", "APPROVED"]}, + "author_id": {"type": ["string", "null"]}, + "reviewer_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string"} + }, + "adjudicator_id": {"type": ["string", "null"]}, + "reviewed_at": {"type": ["string", "null"], "format": "date-time"} + } + }, + "dimensions": { + "type": "object", + "additionalProperties": {"type": "string"} + } + } +} diff --git a/src/evidencebench/cli.py b/src/evidencebench/cli.py index eb8d8a5..0c23c65 100644 --- a/src/evidencebench/cli.py +++ b/src/evidencebench/cli.py @@ -10,6 +10,22 @@ from .runner import run from .scoring import aggregate, score_item from .validation import validate_questions +from .datasets_v4 import load_doctrine_items, load_matter_tasks, read_jsonl as read_jsonl_v4 +from .models_v4 import ( + DoctrineItemScore, + DoctrineResponse, + MatterResponse, + MatterTaskScore, +) +from .runner_v4 import run_doctrine, run_matter +from .release_v4 import build_release_manifest +from .scoring_v4 import score_as_dict, score_doctrine_item, score_matter_task +from .statistics_v4 import summarize_doctrine, summarize_matter, summarize_suite +from .validation_v4 import ( + audit_doctrine_overlap, + validate_doctrine_items, + validate_matter_tasks, +) def _validate(args: argparse.Namespace) -> int: @@ -46,6 +62,132 @@ def _export(args: argparse.Namespace) -> int: return 0 +def _write_or_print(payload: dict, output: str | None) -> None: + rendered = json.dumps(payload, indent=2, sort_keys=True) + "\n" + if output: + Path(output).parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text(rendered) + else: + print(rendered, end="") + + +def _validate_v4(args: argparse.Namespace) -> int: + if args.track == "doctrine": + errors = validate_doctrine_items(args.input, official=args.official) + else: + errors = validate_matter_tasks(args.input, official=args.official) + if errors: + print("\n".join(errors)) + return 1 + print(f"valid v4 {args.track}: {args.input}") + return 0 + + +def _score_v4(args: argparse.Namespace) -> int: + responses = read_jsonl_v4(args.responses) + if args.track == "doctrine": + items = load_doctrine_items(args.input) + by_id = { + record["item_id"]: DoctrineResponse.from_dict(record) + for record in responses + } + scores = [ + score_doctrine_item( + item, + by_id.get( + item.id, + DoctrineResponse(item.id, None, [], [], [], None, "missing"), + ), + ) + for item in items + ] + summary = summarize_doctrine(scores) + else: + entries = load_matter_tasks(args.input) + by_id = { + record["task_id"]: MatterResponse.from_dict(record) + for record in responses + } + scores = [ + score_matter_task( + task, + by_id.get(task.id, MatterResponse(task.id, [], [], "missing")), + ) + for _, task in entries + ] + summary = summarize_matter(scores) + _write_or_print( + { + "schema_version": "4.0", + "track": args.track, + "summary": summary, + "items": [score_as_dict(score) for score in scores], + }, + args.output, + ) + return 0 + + +def _load_score_file(path: str, cls): + payload = json.loads(Path(path).read_text()) + return [cls(**record) for record in payload["items"]] + + +def _summarize_v4(args: argparse.Namespace) -> int: + doctrine = _load_score_file(args.doctrine_scores, DoctrineItemScore) + matter_records = json.loads(Path(args.matter_scores).read_text())["items"] + matter = [ + MatterTaskScore( + **{ + **record, + "criteria": record["criteria"], + } + ) + for record in matter_records + ] + _write_or_print(summarize_suite(doctrine, matter), args.output) + return 0 + + +def _run_v4_doctrine(args: argparse.Namespace) -> int: + _write_or_print( + run_doctrine(args.manifest, args.items, args.output), + None, + ) + return 0 + + +def _run_v4_matter(args: argparse.Namespace) -> int: + _write_or_print( + run_matter(args.manifest, args.tasks, args.output), + None, + ) + return 0 + + +def _audit_v4(args: argparse.Namespace) -> int: + findings = audit_doctrine_overlap(args.public, args.holdout) + if findings: + print("\n".join(findings)) + return 1 + print("no exact stem or family overlap detected") + return 0 + + +def _manifest_v4(args: argparse.Namespace) -> int: + try: + manifest = build_release_manifest( + args.doctrine, + args.matter, + official=args.official, + ) + except ValueError as error: + print(error) + return 1 + _write_or_print(manifest, args.output) + return 0 + + def main() -> int: parser = argparse.ArgumentParser(prog="evidencebench") subparsers = parser.add_subparsers(required=True) @@ -66,6 +208,42 @@ def main() -> int: export.add_argument("--manifest", required=True) export.add_argument("--output", required=True) export.set_defaults(handler=_export) + validate_v4 = subparsers.add_parser("validate-v4") + validate_v4.add_argument("--track", choices=("doctrine", "matter"), required=True) + validate_v4.add_argument("--input", required=True) + validate_v4.add_argument("--official", action="store_true") + validate_v4.set_defaults(handler=_validate_v4) + score_v4 = subparsers.add_parser("score-v4") + score_v4.add_argument("--track", choices=("doctrine", "matter"), required=True) + score_v4.add_argument("--input", required=True) + score_v4.add_argument("--responses", required=True) + score_v4.add_argument("--output") + score_v4.set_defaults(handler=_score_v4) + summarize_v4 = subparsers.add_parser("summarize-v4") + summarize_v4.add_argument("--doctrine-scores", required=True) + summarize_v4.add_argument("--matter-scores", required=True) + summarize_v4.add_argument("--output") + summarize_v4.set_defaults(handler=_summarize_v4) + run_v4_doctrine = subparsers.add_parser("run-v4-doctrine") + run_v4_doctrine.add_argument("--manifest", required=True) + run_v4_doctrine.add_argument("--items", required=True) + run_v4_doctrine.add_argument("--output", required=True) + run_v4_doctrine.set_defaults(handler=_run_v4_doctrine) + run_v4_matter = subparsers.add_parser("run-v4-matter") + run_v4_matter.add_argument("--manifest", required=True) + run_v4_matter.add_argument("--tasks", required=True) + run_v4_matter.add_argument("--output", required=True) + run_v4_matter.set_defaults(handler=_run_v4_matter) + audit_v4 = subparsers.add_parser("audit-v4") + audit_v4.add_argument("--public", required=True) + audit_v4.add_argument("--holdout", required=True) + audit_v4.set_defaults(handler=_audit_v4) + manifest_v4 = subparsers.add_parser("manifest-v4") + manifest_v4.add_argument("--doctrine", required=True) + manifest_v4.add_argument("--matter", required=True) + manifest_v4.add_argument("--output") + manifest_v4.add_argument("--official", action="store_true") + manifest_v4.set_defaults(handler=_manifest_v4) args = parser.parse_args() return args.handler(args) diff --git a/src/evidencebench/datasets_v4.py b/src/evidencebench/datasets_v4.py new file mode 100644 index 0000000..c2982b2 --- /dev/null +++ b/src/evidencebench/datasets_v4.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .models_v4 import DoctrineItem, MatterTask + + +def read_json(path: str | Path) -> dict[str, Any]: + return json.loads(Path(path).read_text()) + + +def read_jsonl(path: str | Path) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in Path(path).read_text().splitlines() + if line.strip() + ] + + +def load_doctrine_items(path: str | Path) -> list[DoctrineItem]: + return [DoctrineItem.from_dict(item) for item in read_jsonl(path)] + + +def load_matter_task(path: str | Path) -> MatterTask: + return MatterTask.from_dict(read_json(path)) + + +def load_matter_tasks(path: str | Path) -> list[tuple[Path, MatterTask]]: + root = Path(path) + if root.is_file(): + return [(root.parent, load_matter_task(root))] + return [ + (task_path.parent, load_matter_task(task_path)) + for task_path in sorted(root.glob("**/task.json")) + ] diff --git a/src/evidencebench/models_v4.py b/src/evidencebench/models_v4.py new file mode 100644 index 0000000..0d6f9db --- /dev/null +++ b/src/evidencebench/models_v4.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class ReviewRecord: + status: str + author_id: str | None = None + reviewer_ids: list[str] = field(default_factory=list) + adjudicator_id: str | None = None + reviewed_at: str | None = None + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "ReviewRecord": + return cls( + status=payload["status"], + author_id=payload.get("author_id"), + reviewer_ids=list(payload.get("reviewer_ids", [])), + adjudicator_id=payload.get("adjudicator_id"), + reviewed_at=payload.get("reviewed_at"), + ) + + +@dataclass(frozen=True) +class Fact: + id: str + text: str + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "Fact": + return cls(id=payload["id"], text=payload["text"]) + + +@dataclass(frozen=True) +class GroundingAnnotation: + issue_code: str + fact_ids: list[str] + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "GroundingAnnotation": + return cls( + issue_code=payload["issue_code"], + fact_ids=list(payload["fact_ids"]), + ) + + +@dataclass(frozen=True) +class DoctrineGold: + ruling: str + issue_codes: list[str] + required_authority_groups: list[list[str]] + accepted_authorities: list[str] + grounding: list[GroundingAnnotation] + rationale: str + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "DoctrineGold": + return cls( + ruling=payload["ruling"], + issue_codes=list(payload["issue_codes"]), + required_authority_groups=[ + list(group) for group in payload["required_authority_groups"] + ], + accepted_authorities=list(payload["accepted_authorities"]), + grounding=[ + GroundingAnnotation.from_dict(item) + for item in payload.get("grounding", []) + ], + rationale=payload["rationale"], + ) + + +@dataclass(frozen=True) +class DoctrineItem: + schema_version: str + track: str + id: str + family_id: str + category: str + difficulty: str + jurisdiction: str + stem: str + facts: list[Fact] + allowed_rulings: list[str] + gold: DoctrineGold + corpus_version: str + review: ReviewRecord + dimensions: dict[str, str] = field(default_factory=dict) + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "DoctrineItem": + return cls( + schema_version=payload["schema_version"], + track=payload["track"], + id=payload["id"], + family_id=payload["family_id"], + category=payload["category"], + difficulty=payload["difficulty"], + jurisdiction=payload["jurisdiction"], + stem=payload["stem"], + facts=[Fact.from_dict(item) for item in payload["facts"]], + allowed_rulings=list(payload["allowed_rulings"]), + gold=DoctrineGold.from_dict(payload["gold"]), + corpus_version=payload["corpus_version"], + review=ReviewRecord.from_dict(payload["review"]), + dimensions=dict(payload.get("dimensions", {})), + ) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class DoctrineGrounding: + issue_code: str + fact_ids: list[str] + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "DoctrineGrounding": + return cls( + issue_code=payload["issue_code"], + fact_ids=list(payload.get("fact_ids", [])), + ) + + +@dataclass(frozen=True) +class DoctrineResponse: + item_id: str + ruling: str | None + issue_codes: list[str] + authorities: list[str] + grounding: list[DoctrineGrounding] + confidence: float | None + status: str = "ok" + explanation: str = "" + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "DoctrineResponse": + return cls( + item_id=payload["item_id"], + ruling=payload.get("ruling"), + issue_codes=list(payload.get("issue_codes", [])), + authorities=list(payload.get("authorities", [])), + grounding=[ + DoctrineGrounding.from_dict(item) + for item in payload.get("grounding", []) + ], + confidence=payload.get("confidence"), + status=payload.get("status", "ok"), + explanation=payload.get("explanation", ""), + ) + + +@dataclass(frozen=True) +class DoctrineItemScore: + item_id: str + family_id: str + outcome_accuracy: float + issue_precision: float + issue_recall: float + issue_f1: float + authority_precision: float + authority_recall: float + authority_f1: float + grounding_precision: float + grounding_recall: float + grounding_f1: float + calibration: float + invalid_authorities: list[str] + hallucinated_authorities: list[str] + unsupported_authorities: list[str] + doctrine_score: float + status: str + + +@dataclass(frozen=True) +class MatterDocument: + path: str + sha256: str + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "MatterDocument": + return cls(path=payload["path"], sha256=payload["sha256"]) + + +@dataclass(frozen=True) +class MatterCriterion: + id: str + dimension: str + title: str + issue_code: str | None = None + expected_disposition: str | None = None + required_fact_ids: list[str] = field(default_factory=list) + required_record_refs: list[str] = field(default_factory=list) + required_authority_groups: list[list[str]] = field(default_factory=list) + accepted_authorities: list[str] = field(default_factory=list) + deliverable: str | None = None + critical: bool = True + review_only: bool = False + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "MatterCriterion": + return cls( + id=payload["id"], + dimension=payload["dimension"], + title=payload["title"], + issue_code=payload.get("issue_code"), + expected_disposition=payload.get("expected_disposition"), + required_fact_ids=list(payload.get("required_fact_ids", [])), + required_record_refs=list(payload.get("required_record_refs", [])), + required_authority_groups=[ + list(group) + for group in payload.get("required_authority_groups", []) + ], + accepted_authorities=list(payload.get("accepted_authorities", [])), + deliverable=payload.get("deliverable"), + critical=payload.get("critical", True), + review_only=payload.get("review_only", False), + ) + + +@dataclass(frozen=True) +class MatterTask: + schema_version: str + track: str + id: str + family_id: str + title: str + task_type: str + instructions: str + documents: list[MatterDocument] + deliverables: list[str] + criteria: list[MatterCriterion] + corpus_version: str + review: ReviewRecord + dimensions: dict[str, str] = field(default_factory=dict) + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "MatterTask": + return cls( + schema_version=payload["schema_version"], + track=payload["track"], + id=payload["id"], + family_id=payload["family_id"], + title=payload["title"], + task_type=payload["task_type"], + instructions=payload["instructions"], + documents=[ + MatterDocument.from_dict(item) for item in payload["documents"] + ], + deliverables=list(payload["deliverables"]), + criteria=[ + MatterCriterion.from_dict(item) for item in payload["criteria"] + ], + corpus_version=payload["corpus_version"], + review=ReviewRecord.from_dict(payload["review"]), + dimensions=dict(payload.get("dimensions", {})), + ) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class MatterFinding: + issue_code: str + disposition: str | None + fact_ids: list[str] + record_refs: list[str] + authorities: list[str] + explanation: str = "" + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "MatterFinding": + return cls( + issue_code=payload["issue_code"], + disposition=payload.get("disposition"), + fact_ids=list(payload.get("fact_ids", [])), + record_refs=list(payload.get("record_refs", [])), + authorities=list(payload.get("authorities", [])), + explanation=payload.get("explanation", ""), + ) + + +@dataclass(frozen=True) +class MatterResponse: + task_id: str + findings: list[MatterFinding] + deliverables: list[str] + status: str = "ok" + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "MatterResponse": + return cls( + task_id=payload["task_id"], + findings=[ + MatterFinding.from_dict(item) + for item in payload.get("findings", []) + ], + deliverables=list(payload.get("deliverables", [])), + status=payload.get("status", "ok"), + ) + + +@dataclass(frozen=True) +class MatterCriterionScore: + criterion_id: str + dimension: str + passed: bool + critical: bool + review_only: bool + + +@dataclass(frozen=True) +class MatterTaskScore: + task_id: str + family_id: str + legal_criteria_rate: float + authority_grounding_rate: float + factual_accuracy_rate: float + deliverable_completeness_rate: float + matter_score: float + complete_task: bool + criteria: list[MatterCriterionScore] + invalid_authorities: list[str] + hallucinated_authorities: list[str] + status: str diff --git a/src/evidencebench/release_v4.py b/src/evidencebench/release_v4.py new file mode 100644 index 0000000..a0b35ba --- /dev/null +++ b/src/evidencebench/release_v4.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + +from .datasets import file_sha256 +from .datasets_v4 import load_doctrine_items, load_matter_tasks +from .validation_v4 import validate_doctrine_items, validate_matter_tasks + + +def _tree_commitment(root: Path) -> tuple[str, list[dict]]: + inventory = [] + for path in sorted(root.rglob("*")): + if path.is_file(): + inventory.append( + { + "path": str(path.relative_to(root)), + "sha256": file_sha256(str(path)), + "bytes": path.stat().st_size, + } + ) + canonical = json.dumps(inventory, separators=(",", ":"), sort_keys=True).encode() + return hashlib.sha256(canonical).hexdigest(), inventory + + +def build_release_manifest( + doctrine_path: str | Path, + matter_path: str | Path, + *, + official: bool, +) -> dict: + doctrine_errors = validate_doctrine_items(doctrine_path, official=official) + matter_errors = validate_matter_tasks(matter_path, official=official) + if doctrine_errors or matter_errors: + raise ValueError("\n".join(doctrine_errors + matter_errors)) + doctrine = load_doctrine_items(doctrine_path) + matters = load_matter_tasks(matter_path) + matter_root = Path(matter_path) + matter_hash, inventory = _tree_commitment(matter_root) + protocol_root = Path(__file__).resolve().parent + protocol_files = [ + protocol_root / "models_v4.py", + protocol_root / "scoring_v4.py", + protocol_root / "statistics_v4.py", + protocol_root / "validation_v4.py", + protocol_root / "runner_v4.py", + ] + protocol_inventory = [ + { + "path": path.name, + "sha256": file_sha256(str(path)), + } + for path in protocol_files + ] + protocol_hash = hashlib.sha256( + json.dumps( + protocol_inventory, separators=(",", ":"), sort_keys=True + ).encode() + ).hexdigest() + return { + "schema_version": "4.0", + "release_status": "official" if official else "development", + "generated_at": datetime.now(timezone.utc).isoformat(), + "headline_formula": {"doctrine": 0.5, "matter": 0.5}, + "doctrine": { + "sha256": file_sha256(str(doctrine_path)), + "count": len(doctrine), + "family_count": len({item.family_id for item in doctrine}), + "categories": dict(sorted(Counter(item.category for item in doctrine).items())), + "review_statuses": dict( + sorted(Counter(item.review.status for item in doctrine).items()) + ), + }, + "matter": { + "tree_sha256": matter_hash, + "count": len(matters), + "family_count": len({task.family_id for _, task in matters}), + "task_types": dict( + sorted(Counter(task.task_type for _, task in matters).items()) + ), + "review_statuses": dict( + sorted(Counter(task.review.status for _, task in matters).items()) + ), + "inventory": inventory, + }, + "protocol": { + "sha256": protocol_hash, + "files": protocol_inventory, + }, + "authority_corpus": "fre-2025", + } diff --git a/src/evidencebench/runner_v4.py b/src/evidencebench/runner_v4.py new file mode 100644 index 0000000..5fc784f --- /dev/null +++ b/src/evidencebench/runner_v4.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import json +import os +import re +import socket +import subprocess +import urllib.error +import urllib.request +import zipfile +from pathlib import Path +from xml.etree import ElementTree + +from .datasets import file_sha256 +from .datasets_v4 import load_doctrine_items, load_matter_tasks +from .models_v4 import MatterTask + + +DOCTRINE_PROMPT_VERSION = "evidencebench-v4-doctrine-1" +MATTER_PROMPT_VERSION = "evidencebench-v4-matter-agent-1" +DEFAULT_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" + + +def _load_ignored_env(start: Path) -> None: + path = None + for directory in (start, *start.parents): + candidate = directory / ".env" + if candidate.is_file(): + path = candidate + break + if (directory / ".git").exists(): + break + if path is None: + return + for line in path.read_text().splitlines(): + match = re.match( + r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$", + line, + ) + if not match: + continue + name, value = match.groups() + if value[:1] == value[-1:] and value[:1] in {"'", '"'}: + value = value[1:-1] + os.environ.setdefault(name, value) + + +def _post_openrouter(manifest: dict, payload: dict) -> dict: + api_key_env = manifest.get("api_key_env", "OPENROUTER_API_KEY") + api_key = os.environ.get(api_key_env) + if not api_key: + raise RuntimeError(f"missing required environment variable: {api_key_env}") + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "X-Title": manifest.get("app_title", "EvidenceBench v4"), + } + if manifest.get("http_referer"): + headers["HTTP-Referer"] = manifest["http_referer"] + request = urllib.request.Request( + manifest.get("base_url", DEFAULT_OPENROUTER_URL), + data=json.dumps(payload).encode(), + headers=headers, + method="POST", + ) + with urllib.request.urlopen(request, timeout=manifest.get("timeout_seconds", 180)) as response: + return json.loads(response.read()) + + +def _is_retryable(error: Exception) -> bool: + if isinstance(error, (urllib.error.URLError, TimeoutError, socket.timeout)): + return True + return isinstance(error, urllib.error.HTTPError) and error.code in { + 408, + 429, + 500, + 502, + 503, + 504, + } + + +def _request(manifest: dict, payload: dict) -> dict: + for attempt in range(2): + try: + return _post_openrouter(manifest, payload) + except Exception as error: + if attempt == 0 and _is_retryable(error): + continue + raise + raise AssertionError("unreachable") + + +def _json_content(message: dict) -> dict: + content = message.get("content", "") + if isinstance(content, list): + content = "".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) + if not isinstance(content, str): + raise ValueError("model response content must be text") + content = re.sub(r"^```(?:json)?\s*|\s*```$", "", content.strip()) + parsed = json.loads(content) + if not isinstance(parsed, dict): + raise ValueError("model response must contain one JSON object") + return parsed + + +def doctrine_prompt(item) -> str: + facts = "\n".join(f"{fact.id}: {fact.text}" for fact in item.facts) + rulings = ", ".join(item.allowed_rulings) + return f"""You are completing the EvidenceBench v4 Doctrine track. +This is closed-book. Do not browse or use tools. Apply the Federal Rules of +Evidence and return one JSON object only. Use only the supplied fact IDs. + +Required shape: +{{"ruling":"admit|exclude|limit|defer","issue_codes":["RULE_..."], +"authorities":["FRE 123(a)"],"grounding":[{{"issue_code":"RULE_...", +"fact_ids":["F1"]}}],"confidence":0.0,"explanation":"brief analysis"}} + +Allowed rulings: {rulings} +Question: {item.stem} +Facts: +{facts} +""" + + +def run_doctrine( + manifest_path: str | Path, + items_path: str | Path, + output_path: str | Path, +) -> dict: + manifest_path = Path(manifest_path) + _load_ignored_env(manifest_path.resolve().parent) + manifest = json.loads(manifest_path.read_text()) + if manifest.get("provider") != "openrouter": + raise ValueError("v4 runner requires provider=openrouter") + if manifest.get("tools_enabled"): + raise ValueError("Doctrine track must disable tools") + outputs = [] + for item in load_doctrine_items(items_path): + try: + result = _request( + manifest, + { + "model": manifest["model_id"], + "temperature": 0, + "seed": manifest.get("seed", 20260304), + "max_tokens": manifest.get("max_output_tokens", 1200), + "response_format": {"type": "json_object"}, + "messages": [{"role": "user", "content": doctrine_prompt(item)}], + }, + ) + parsed = _json_content(result["choices"][0]["message"]) + outputs.append( + { + **parsed, + "item_id": item.id, + "status": "ok", + "usage": result.get("usage", {}), + "generation_id": result.get("id"), + } + ) + except Exception as error: + outputs.append( + { + "item_id": item.id, + "ruling": None, + "issue_codes": [], + "authorities": [], + "grounding": [], + "confidence": None, + "explanation": "", + "status": f"failed:{type(error).__name__}", + } + ) + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + "\n".join(json.dumps(record, sort_keys=True) for record in outputs) + "\n" + ) + return { + "provider": "openrouter", + "model": manifest["model_id"], + "prompt_version": DOCTRINE_PROMPT_VERSION, + "dataset_sha256": file_sha256(str(items_path)), + "output_path": str(output), + } + + +def _safe_path(root: Path, relative: str) -> Path: + if Path(relative).is_absolute(): + raise ValueError("absolute paths are not allowed") + candidate = (root / relative).resolve() + try: + candidate.relative_to(root.resolve()) + except ValueError as exc: + raise ValueError("path escapes the workspace") from exc + return candidate + + +def _extract_document(path: Path) -> str: + if path.suffix.casefold() in {".txt", ".md", ".json", ".csv", ".tsv"}: + return path.read_text(errors="replace") + if path.suffix.casefold() == ".pdf": + completed = subprocess.run( + ["pdftotext", "-layout", str(path), "-"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + return completed.stdout + if path.suffix.casefold() in {".docx", ".pptx", ".xlsx"}: + chunks: list[str] = [] + with zipfile.ZipFile(path) as archive: + for name in sorted(archive.namelist()): + if not name.endswith(".xml"): + continue + try: + root = ElementTree.fromstring(archive.read(name)) + except ElementTree.ParseError: + continue + text = " ".join( + node.text.strip() + for node in root.iter() + if node.text and node.text.strip() + ) + if text: + chunks.append(f"[{name}]\n{text}") + return "\n\n".join(chunks) + raise ValueError(f"unsupported document format: {path.suffix}") + + +MATTER_TOOLS = [ + { + "type": "function", + "function": { + "name": "list_documents", + "description": "List matter documents available for review.", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "read_document", + "description": "Read a matter document by its relative path.", + "parameters": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_documents", + "description": "Search all text-extractable documents for a literal query.", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "write_output", + "description": "Write a UTF-8 output file. findings.json is mandatory.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "content": {"type": "string"}, + }, + "required": ["path", "content"], + }, + }, + }, +] + + +class MatterWorkspace: + def __init__(self, document_root: Path, output_root: Path) -> None: + self.document_root = document_root.resolve() + self.output_root = output_root.resolve() + self.output_root.mkdir(parents=True, exist_ok=True) + + def execute(self, name: str, arguments: dict) -> dict: + if name == "list_documents": + return { + "documents": [ + str(path.relative_to(self.document_root)) + for path in sorted(self.document_root.rglob("*")) + if path.is_file() + ] + } + if name == "read_document": + path = _safe_path(self.document_root, arguments["path"]) + return {"path": arguments["path"], "content": _extract_document(path)} + if name == "search_documents": + query = arguments["query"].casefold() + matches = [] + for path in sorted(self.document_root.rglob("*")): + if not path.is_file(): + continue + try: + text = _extract_document(path) + except (ValueError, OSError, subprocess.SubprocessError): + continue + for number, line in enumerate(text.splitlines(), 1): + if query in line.casefold(): + matches.append( + { + "path": str(path.relative_to(self.document_root)), + "line": number, + "text": line[:500], + } + ) + if len(matches) == 100: + return {"matches": matches, "truncated": True} + return {"matches": matches, "truncated": False} + if name == "write_output": + path = _safe_path(self.output_root, arguments["path"]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(arguments["content"]) + return {"path": arguments["path"], "bytes": len(arguments["content"].encode())} + raise ValueError(f"unknown tool: {name}") + + +def matter_prompt(task: MatterTask) -> str: + deliverables = ", ".join(task.deliverables) + return f"""You are completing the EvidenceBench v4 Matter track in a +restricted workspace. Review the supplied record with the available tools. +Do not browse or assume facts not in the record. + +Task: {task.title} +Instructions: {task.instructions} +Required deliverables: {deliverables} + +You must write findings.json with this shape: +{{"findings":[{{"issue_code":"...","disposition":"...", +"fact_ids":["..."],"record_refs":["document:line-or-section"], +"authorities":["FRE 123(a)"],"explanation":"..."}}]}} +Use write_output for every deliverable. When complete, respond briefly. +""" + + +def _run_matter_task( + manifest: dict, + task_dir: Path, + task: MatterTask, + output_root: Path, +) -> tuple[dict, list[dict], dict]: + workspace = MatterWorkspace(task_dir / "documents", output_root) + messages: list[dict] = [{"role": "user", "content": matter_prompt(task)}] + transcript: list[dict] = list(messages) + total_usage: dict[str, int] = {} + for _ in range(manifest.get("max_turns", 20)): + result = _request( + manifest, + { + "model": manifest["model_id"], + "temperature": 0, + "seed": manifest.get("seed", 20260304), + "max_tokens": manifest.get("max_output_tokens", 2000), + "messages": messages, + "tools": MATTER_TOOLS, + "parallel_tool_calls": False, + }, + ) + for key, value in result.get("usage", {}).items(): + if isinstance(value, int): + total_usage[key] = total_usage.get(key, 0) + value + message = result["choices"][0]["message"] + messages.append(message) + transcript.append(message) + tool_calls = message.get("tool_calls", []) + if not tool_calls: + break + for tool_call in tool_calls: + function = tool_call["function"] + try: + arguments = json.loads(function.get("arguments", "{}")) + payload = workspace.execute(function["name"], arguments) + except Exception as error: + payload = {"error": f"{type(error).__name__}: {error}"} + tool_message = { + "role": "tool", + "tool_call_id": tool_call["id"], + "name": function["name"], + "content": json.dumps(payload, sort_keys=True), + } + messages.append(tool_message) + transcript.append(tool_message) + + findings_path = output_root / "findings.json" + if not findings_path.is_file(): + raise ValueError("agent did not create findings.json") + parsed = json.loads(findings_path.read_text()) + if not isinstance(parsed.get("findings"), list): + raise ValueError("findings.json must contain a findings list") + actual_deliverables = [ + str(path.relative_to(output_root)) + for path in sorted(output_root.rglob("*")) + if path.is_file() + ] + response = { + "task_id": task.id, + "findings": parsed["findings"], + "deliverables": actual_deliverables, + "status": "ok", + } + return response, transcript, total_usage + + +def run_matter( + manifest_path: str | Path, + tasks_path: str | Path, + output_root: str | Path, +) -> dict: + manifest_path = Path(manifest_path) + _load_ignored_env(manifest_path.resolve().parent) + manifest = json.loads(manifest_path.read_text()) + if manifest.get("provider") != "openrouter": + raise ValueError("v4 runner requires provider=openrouter") + if not manifest.get("tools_enabled"): + raise ValueError("Matter track requires tools_enabled=true") + destination = Path(output_root) + destination.mkdir(parents=True, exist_ok=True) + records = [] + for task_dir, task in load_matter_tasks(tasks_path): + task_output = destination / "workspaces" / task.id + if task_output.exists(): + raise FileExistsError( + f"refusing to overwrite prior task workspace: {task_output}" + ) + task_output.mkdir(parents=True) + try: + response, transcript, usage = _run_matter_task( + manifest, task_dir, task, task_output + ) + except Exception as error: + response = { + "task_id": task.id, + "findings": [], + "deliverables": [], + "status": f"failed:{type(error).__name__}", + } + transcript = [] + usage = {} + records.append(response) + (destination / f"{task.id}.transcript.json").write_text( + json.dumps({"messages": transcript, "usage": usage}, indent=2) + "\n" + ) + response_path = destination / "responses.jsonl" + response_path.write_text( + "\n".join(json.dumps(record, sort_keys=True) for record in records) + "\n" + ) + return { + "provider": "openrouter", + "model": manifest["model_id"], + "prompt_version": MATTER_PROMPT_VERSION, + "response_path": str(response_path), + "tasks": len(records), + } diff --git a/src/evidencebench/scoring_v4.py b/src/evidencebench/scoring_v4.py new file mode 100644 index 0000000..d7cdc16 --- /dev/null +++ b/src/evidencebench/scoring_v4.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +from collections import defaultdict +from dataclasses import asdict +from typing import Iterable + +from .citations import citation_exists, normalize_citation +from .models_v4 import ( + DoctrineItem, + DoctrineItemScore, + DoctrineResponse, + MatterCriterion, + MatterCriterionScore, + MatterFinding, + MatterResponse, + MatterTask, + MatterTaskScore, +) + + +DOCTRINE_WEIGHTS = { + "outcome_accuracy": 0.40, + "issue_f1": 0.25, + "authority_f1": 0.20, + "grounding_f1": 0.10, + "calibration": 0.05, +} + +MATTER_WEIGHTS = { + "legal": 0.50, + "authority": 0.20, + "fact": 0.15, + "deliverable": 0.15, +} + + +def _normalized_codes(values: Iterable[str]) -> set[str]: + return {value.strip().upper() for value in values if value.strip()} + + +def _precision_recall_f1( + predicted: set[object], expected: set[object] +) -> tuple[float, float, float]: + true_positive = len(predicted.intersection(expected)) + precision = true_positive / len(predicted) if predicted else 0.0 + recall = true_positive / len(expected) if expected else 1.0 + f1 = ( + 0.0 + if precision + recall == 0 + else 2 * precision * recall / (precision + recall) + ) + return precision, recall, f1 + + +def _authority_score( + submitted: Iterable[str], + accepted_values: Iterable[str], + required_groups: Iterable[Iterable[str]], +) -> tuple[float, float, float, list[str], list[str], list[str]]: + canonical_submissions: dict[str, str] = {} + invalid: list[str] = [] + for raw_value in submitted: + raw = raw_value.strip() + if not raw: + continue + normalized = normalize_citation(raw) + if normalized is None: + invalid.append(raw) + canonical_submissions.setdefault(f"INVALID::{raw.casefold()}", raw) + else: + canonical_submissions.setdefault(normalized, raw) + + accepted = { + normalized + for value in accepted_values + if (normalized := normalize_citation(value)) is not None + } + supported = { + value for value in canonical_submissions if value in accepted + } + hallucinated = sorted( + original + for normalized, original in canonical_submissions.items() + if not normalized.startswith("INVALID::") + and normalized not in accepted + and not citation_exists(normalized) + ) + unsupported = sorted( + original + for normalized, original in canonical_submissions.items() + if not normalized.startswith("INVALID::") + and normalized not in accepted + and citation_exists(normalized) + ) + precision = ( + len(supported) / len(canonical_submissions) + if canonical_submissions + else 0.0 + ) + normalized_groups = [ + { + normalized + for value in group + if (normalized := normalize_citation(value)) is not None + } + for group in required_groups + ] + recall = ( + sum(bool(group.intersection(supported)) for group in normalized_groups) + / len(normalized_groups) + if normalized_groups + else 1.0 + ) + f1 = ( + 0.0 + if precision + recall == 0 + else 2 * precision * recall / (precision + recall) + ) + return precision, recall, f1, sorted(set(invalid)), hallucinated, unsupported + + +def score_doctrine_item( + item: DoctrineItem, response: DoctrineResponse +) -> DoctrineItemScore: + if response.status != "ok": + return DoctrineItemScore( + item_id=item.id, + family_id=item.family_id, + outcome_accuracy=0.0, + issue_precision=0.0, + issue_recall=0.0, + issue_f1=0.0, + authority_precision=0.0, + authority_recall=0.0, + authority_f1=0.0, + grounding_precision=0.0, + grounding_recall=0.0, + grounding_f1=0.0, + calibration=0.0, + invalid_authorities=[], + hallucinated_authorities=[], + unsupported_authorities=[], + doctrine_score=0.0, + status=response.status, + ) + + outcome = float(response.ruling == item.gold.ruling) + issue_precision, issue_recall, issue_f1 = _precision_recall_f1( + _normalized_codes(response.issue_codes), + _normalized_codes(item.gold.issue_codes), + ) + ( + authority_precision, + authority_recall, + authority_f1, + invalid, + hallucinated, + unsupported, + ) = _authority_score( + response.authorities, + item.gold.accepted_authorities, + item.gold.required_authority_groups, + ) + predicted_grounding = { + (entry.issue_code.strip().upper(), fact_id.strip().upper()) + for entry in response.grounding + for fact_id in entry.fact_ids + if entry.issue_code.strip() and fact_id.strip() + } + expected_grounding = { + (entry.issue_code.strip().upper(), fact_id.strip().upper()) + for entry in item.gold.grounding + for fact_id in entry.fact_ids + } + grounding_precision, grounding_recall, grounding_f1 = ( + _precision_recall_f1(predicted_grounding, expected_grounding) + ) + confidence = ( + min(1.0, max(0.0, response.confidence)) + if isinstance(response.confidence, (int, float)) + else 0.0 + ) + calibration = 1.0 - (confidence - outcome) ** 2 + doctrine_score = sum( + ( + DOCTRINE_WEIGHTS["outcome_accuracy"] * outcome, + DOCTRINE_WEIGHTS["issue_f1"] * issue_f1, + DOCTRINE_WEIGHTS["authority_f1"] * authority_f1, + DOCTRINE_WEIGHTS["grounding_f1"] * grounding_f1, + DOCTRINE_WEIGHTS["calibration"] * calibration, + ) + ) + return DoctrineItemScore( + item_id=item.id, + family_id=item.family_id, + outcome_accuracy=outcome, + issue_precision=issue_precision, + issue_recall=issue_recall, + issue_f1=issue_f1, + authority_precision=authority_precision, + authority_recall=authority_recall, + authority_f1=authority_f1, + grounding_precision=grounding_precision, + grounding_recall=grounding_recall, + grounding_f1=grounding_f1, + calibration=calibration, + invalid_authorities=invalid, + hallucinated_authorities=hallucinated, + unsupported_authorities=unsupported, + doctrine_score=doctrine_score, + status="ok", + ) + + +def _matching_finding( + criterion: MatterCriterion, findings: list[MatterFinding] +) -> MatterFinding | None: + if criterion.issue_code is None: + return None + expected = criterion.issue_code.strip().upper() + for finding in findings: + if finding.issue_code.strip().upper() != expected: + continue + if ( + criterion.expected_disposition is not None + and finding.disposition != criterion.expected_disposition + ): + continue + return finding + return None + + +def _criterion_passes( + criterion: MatterCriterion, + response: MatterResponse, +) -> bool: + if criterion.review_only: + return False + if criterion.dimension == "deliverable": + return bool( + criterion.deliverable + and criterion.deliverable in set(response.deliverables) + ) + + finding = _matching_finding(criterion, response.findings) + if finding is None: + return False + if criterion.dimension == "legal": + return True + if criterion.dimension == "authority": + _, recall, _, _, _, _ = _authority_score( + finding.authorities, + criterion.accepted_authorities, + criterion.required_authority_groups, + ) + return recall == 1.0 + if criterion.dimension == "fact": + required_facts = _normalized_codes(criterion.required_fact_ids) + submitted_refs = {value.strip() for value in finding.record_refs} + required_refs_pass = all( + any( + submitted == required + or submitted.startswith(f"{required}:") + or submitted.startswith(f"{required}#") + for submitted in submitted_refs + ) + for required in criterion.required_record_refs + ) + return required_facts.issubset(_normalized_codes(finding.fact_ids)) and ( + required_refs_pass + ) + raise ValueError(f"unsupported matter criterion dimension: {criterion.dimension}") + + +def score_matter_task( + task: MatterTask, response: MatterResponse +) -> MatterTaskScore: + if response.status != "ok": + return MatterTaskScore( + task_id=task.id, + family_id=task.family_id, + legal_criteria_rate=0.0, + authority_grounding_rate=0.0, + factual_accuracy_rate=0.0, + deliverable_completeness_rate=0.0, + matter_score=0.0, + complete_task=False, + criteria=[], + invalid_authorities=[], + hallucinated_authorities=[], + status=response.status, + ) + + criterion_scores = [ + MatterCriterionScore( + criterion_id=criterion.id, + dimension=criterion.dimension, + passed=_criterion_passes(criterion, response), + critical=criterion.critical, + review_only=criterion.review_only, + ) + for criterion in task.criteria + ] + by_dimension: dict[str, list[MatterCriterionScore]] = defaultdict(list) + for score in criterion_scores: + if not score.review_only: + by_dimension[score.dimension].append(score) + + def rate(dimension: str) -> float: + values = by_dimension[dimension] + return sum(score.passed for score in values) / len(values) if values else 0.0 + + legal = rate("legal") + authority = rate("authority") + fact = rate("fact") + deliverable = rate("deliverable") + matter_score = ( + MATTER_WEIGHTS["legal"] * legal + + MATTER_WEIGHTS["authority"] * authority + + MATTER_WEIGHTS["fact"] * fact + + MATTER_WEIGHTS["deliverable"] * deliverable + ) + critical_scores = [ + score + for score in criterion_scores + if score.critical and not score.review_only + ] + all_authorities = [ + authority_value + for finding in response.findings + for authority_value in finding.authorities + ] + accepted = [ + authority_value + for criterion in task.criteria + for authority_value in criterion.accepted_authorities + ] + _, _, _, invalid, hallucinated, _ = _authority_score( + all_authorities, accepted, [] + ) + return MatterTaskScore( + task_id=task.id, + family_id=task.family_id, + legal_criteria_rate=legal, + authority_grounding_rate=authority, + factual_accuracy_rate=fact, + deliverable_completeness_rate=deliverable, + matter_score=matter_score, + complete_task=bool(critical_scores) + and all(score.passed for score in critical_scores), + criteria=criterion_scores, + invalid_authorities=invalid, + hallucinated_authorities=hallucinated, + status="ok", + ) + + +def score_as_dict(score: DoctrineItemScore | MatterTaskScore) -> dict: + return asdict(score) diff --git a/src/evidencebench/statistics_v4.py b/src/evidencebench/statistics_v4.py new file mode 100644 index 0000000..5618882 --- /dev/null +++ b/src/evidencebench/statistics_v4.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import random +from collections import defaultdict +from dataclasses import asdict +from statistics import mean +from typing import Callable, Iterable, TypeVar + +from .models_v4 import DoctrineItemScore, MatterTaskScore + + +T = TypeVar("T") + + +def _percentile(sorted_values: list[float], probability: float) -> float: + if not sorted_values: + return 0.0 + index = (len(sorted_values) - 1) * probability + lower = int(index) + upper = min(lower + 1, len(sorted_values) - 1) + fraction = index - lower + return sorted_values[lower] * (1 - fraction) + sorted_values[upper] * fraction + + +def cluster_bootstrap_interval( + values: Iterable[T], + family: Callable[[T], str], + metric: Callable[[T], float], + *, + iterations: int = 5000, + seed: int = 20260304, +) -> tuple[float, float]: + grouped: dict[str, list[T]] = defaultdict(list) + for value in values: + grouped[family(value)].append(value) + families = sorted(grouped) + if not families: + return 0.0, 0.0 + generator = random.Random(seed) + estimates: list[float] = [] + for _ in range(iterations): + sampled = [ + item + for _ in families + for item in grouped[generator.choice(families)] + ] + estimates.append(mean(metric(item) for item in sampled)) + estimates.sort() + return _percentile(estimates, 0.025), _percentile(estimates, 0.975) + + +def summarize_doctrine(scores: list[DoctrineItemScore]) -> dict: + metric_names = ( + "doctrine_score", + "outcome_accuracy", + "issue_f1", + "authority_f1", + "grounding_f1", + "calibration", + ) + result = { + "n": len(scores), + **{ + name: mean(getattr(score, name) for score in scores) if scores else 0.0 + for name in metric_names + }, + "invalid_authority_count": sum( + len(score.invalid_authorities) for score in scores + ), + "hallucinated_authority_count": sum( + len(score.hallucinated_authorities) for score in scores + ), + "failure_count": sum(score.status != "ok" for score in scores), + } + low, high = cluster_bootstrap_interval( + scores, lambda score: score.family_id, lambda score: score.doctrine_score + ) + result["score_ci_95"] = [low, high] + return result + + +def summarize_matter(scores: list[MatterTaskScore]) -> dict: + metric_names = ( + "matter_score", + "legal_criteria_rate", + "authority_grounding_rate", + "factual_accuracy_rate", + "deliverable_completeness_rate", + ) + result = { + "n": len(scores), + **{ + name: mean(getattr(score, name) for score in scores) if scores else 0.0 + for name in metric_names + }, + "complete_task_rate": mean(score.complete_task for score in scores) + if scores + else 0.0, + "invalid_authority_count": sum( + len(score.invalid_authorities) for score in scores + ), + "hallucinated_authority_count": sum( + len(score.hallucinated_authorities) for score in scores + ), + "failure_count": sum(score.status != "ok" for score in scores), + } + low, high = cluster_bootstrap_interval( + scores, lambda score: score.family_id, lambda score: score.matter_score + ) + result["score_ci_95"] = [low, high] + return result + + +def summarize_suite( + doctrine_scores: list[DoctrineItemScore], + matter_scores: list[MatterTaskScore], +) -> dict: + doctrine = summarize_doctrine(doctrine_scores) + matter = summarize_matter(matter_scores) + if not doctrine_scores or not matter_scores: + raise ValueError("overall score requires non-empty Doctrine and Matter tracks") + overall = 0.5 * doctrine["doctrine_score"] + 0.5 * matter["matter_score"] + + doctrine_groups: dict[str, list[DoctrineItemScore]] = defaultdict(list) + matter_groups: dict[str, list[MatterTaskScore]] = defaultdict(list) + for score in doctrine_scores: + doctrine_groups[score.family_id].append(score) + for score in matter_scores: + matter_groups[score.family_id].append(score) + doctrine_families = sorted(doctrine_groups) + matter_families = sorted(matter_groups) + generator = random.Random(20260304) + samples: list[float] = [] + for _ in range(5000): + sampled_doctrine = [ + item + for _ in doctrine_families + for item in doctrine_groups[generator.choice(doctrine_families)] + ] + sampled_matter = [ + item + for _ in matter_families + for item in matter_groups[generator.choice(matter_families)] + ] + samples.append( + 0.5 * mean(score.doctrine_score for score in sampled_doctrine) + + 0.5 * mean(score.matter_score for score in sampled_matter) + ) + samples.sort() + return { + "schema_version": "4.0", + "overall_score": overall, + "overall_score_100": overall * 100, + "overall_ci_95": [ + _percentile(samples, 0.025), + _percentile(samples, 0.975), + ], + "track_weights": {"doctrine": 0.5, "matter": 0.5}, + "weight_sensitivity": { + "doctrine_40_matter_60": ( + 0.4 * doctrine["doctrine_score"] + 0.6 * matter["matter_score"] + ), + "doctrine_50_matter_50": overall, + "doctrine_60_matter_40": ( + 0.6 * doctrine["doctrine_score"] + 0.4 * matter["matter_score"] + ), + }, + "doctrine": doctrine, + "matter": matter, + } + + +def score_records(scores: Iterable[DoctrineItemScore | MatterTaskScore]) -> list[dict]: + return [asdict(score) for score in scores] diff --git a/src/evidencebench/validation_v4.py b/src/evidencebench/validation_v4.py new file mode 100644 index 0000000..2ff5d5e --- /dev/null +++ b/src/evidencebench/validation_v4.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import hashlib +import re +from collections import Counter +from pathlib import Path + +from .citations import citation_exists, normalize_citation +from .datasets_v4 import load_doctrine_items, load_matter_tasks +from .models_v4 import MatterTask, ReviewRecord + + +SCHEMA_VERSION = "4.0" +REVIEW_STATUSES = {"DRAFT", "IN_REVIEW", "APPROVED"} +RULINGS = {"admit", "exclude", "limit", "defer"} +MATTER_DIMENSIONS = {"legal", "authority", "fact", "deliverable"} + + +def _review_errors(review: ReviewRecord, label: str, official: bool) -> list[str]: + errors: list[str] = [] + if review.status not in REVIEW_STATUSES: + errors.append(f"{label}: invalid review status {review.status!r}") + if official: + if review.status != "APPROVED": + errors.append(f"{label}: official data must be APPROVED") + if not review.author_id: + errors.append(f"{label}: official data requires author_id") + if not review.reviewer_ids: + errors.append(f"{label}: official data requires at least one reviewer") + if review.author_id and review.author_id in review.reviewer_ids: + errors.append(f"{label}: author cannot be the sole independent reviewer") + if not review.reviewed_at: + errors.append(f"{label}: official data requires reviewed_at") + return errors + + +def _authority_errors(values: list[str], label: str) -> list[str]: + errors: list[str] = [] + for value in values: + normalized = normalize_citation(value) + if normalized is None: + errors.append(f"{label}: unparseable authority {value!r}") + elif not citation_exists(normalized): + errors.append(f"{label}: authority absent from pinned corpus {value!r}") + return errors + + +def _normalized_text(value: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", value.casefold())).strip() + + +def validate_doctrine_items(path: str | Path, official: bool = False) -> list[str]: + errors: list[str] = [] + try: + items = load_doctrine_items(path) + except (KeyError, TypeError, ValueError) as exc: + return [f"{path}: could not parse doctrine data: {exc}"] + ids: Counter[str] = Counter(item.id for item in items) + duplicate_ids = sorted(item_id for item_id, count in ids.items() if count > 1) + for item_id in duplicate_ids: + errors.append(f"duplicate doctrine id: {item_id}") + + normalized_stems: dict[str, str] = {} + for item in items: + label = item.id + if item.schema_version != SCHEMA_VERSION: + errors.append(f"{label}: schema_version must be {SCHEMA_VERSION}") + if item.track != "doctrine": + errors.append(f"{label}: track must be doctrine") + if not item.family_id: + errors.append(f"{label}: family_id is required") + if item.gold.ruling not in item.allowed_rulings: + errors.append(f"{label}: gold ruling is not in allowed_rulings") + if set(item.allowed_rulings) - RULINGS: + errors.append(f"{label}: unsupported ruling in allowed_rulings") + if not item.gold.issue_codes: + errors.append(f"{label}: at least one gold issue code is required") + + fact_ids = [fact.id for fact in item.facts] + if len(fact_ids) != len(set(fact_ids)): + errors.append(f"{label}: fact ids must be unique") + issue_codes = set(item.gold.issue_codes) + for grounding in item.gold.grounding: + if grounding.issue_code not in issue_codes: + errors.append( + f"{label}: grounding issue {grounding.issue_code!r} is not gold" + ) + missing = sorted(set(grounding.fact_ids) - set(fact_ids)) + if missing: + errors.append(f"{label}: grounding references missing facts {missing}") + + accepted = { + normalize_citation(value) for value in item.gold.accepted_authorities + } + errors.extend( + _authority_errors(item.gold.accepted_authorities, f"{label} accepted") + ) + for group_index, group in enumerate(item.gold.required_authority_groups): + if not group: + errors.append(f"{label}: authority group {group_index} is empty") + errors.extend( + _authority_errors(group, f"{label} authority group {group_index}") + ) + if any(normalize_citation(value) not in accepted for value in group): + errors.append( + f"{label}: authority group {group_index} is not a subset " + "of accepted_authorities" + ) + errors.extend(_review_errors(item.review, label, official)) + + normalized = _normalized_text(item.stem) + if normalized in normalized_stems: + errors.append( + f"{label}: exact normalized stem duplicate of " + f"{normalized_stems[normalized]}" + ) + else: + normalized_stems[normalized] = label + return errors + + +def _document_errors(task: MatterTask, task_dir: Path) -> list[str]: + errors: list[str] = [] + document_root = (task_dir / "documents").resolve() + for document in task.documents: + candidate = (document_root / document.path).resolve() + try: + candidate.relative_to(document_root) + except ValueError: + errors.append(f"{task.id}: unsafe document path {document.path!r}") + continue + if not candidate.is_file(): + errors.append(f"{task.id}: missing document {document.path!r}") + continue + digest = hashlib.sha256(candidate.read_bytes()).hexdigest() + if digest != document.sha256: + errors.append(f"{task.id}: sha256 mismatch for {document.path!r}") + return errors + + +def validate_matter_tasks(path: str | Path, official: bool = False) -> list[str]: + errors: list[str] = [] + root = Path(path) + try: + entries = load_matter_tasks(root) + except (KeyError, TypeError, ValueError) as exc: + return [f"{path}: could not parse matter data: {exc}"] + ids = Counter(task.id for _, task in entries) + for task_id, count in ids.items(): + if count > 1: + errors.append(f"duplicate matter task id: {task_id}") + + for task_dir, task in entries: + label = task.id + if task.schema_version != SCHEMA_VERSION: + errors.append(f"{label}: schema_version must be {SCHEMA_VERSION}") + if task.track != "matter": + errors.append(f"{label}: track must be matter") + if not task.family_id: + errors.append(f"{label}: family_id is required") + if not task.documents: + errors.append(f"{label}: at least one matter document is required") + if not task.deliverables: + errors.append(f"{label}: at least one deliverable is required") + if any(Path(value).is_absolute() or ".." in Path(value).parts for value in task.deliverables): + errors.append(f"{label}: deliverables must use safe relative paths") + + criterion_ids = [criterion.id for criterion in task.criteria] + if len(criterion_ids) != len(set(criterion_ids)): + errors.append(f"{label}: criterion ids must be unique") + scoring = [criterion for criterion in task.criteria if not criterion.review_only] + dimensions = {criterion.dimension for criterion in scoring} + missing_dimensions = MATTER_DIMENSIONS - dimensions + if missing_dimensions: + errors.append( + f"{label}: missing scoring dimensions {sorted(missing_dimensions)}" + ) + if any(criterion.dimension not in MATTER_DIMENSIONS for criterion in scoring): + errors.append(f"{label}: unsupported scoring criterion dimension") + if not any(criterion.critical for criterion in scoring): + errors.append(f"{label}: at least one scoring criterion must be critical") + for criterion in scoring: + criterion_label = f"{label}/{criterion.id}" + if criterion.dimension == "deliverable": + if criterion.deliverable not in task.deliverables: + errors.append( + f"{criterion_label}: deliverable is not declared by task" + ) + else: + if not criterion.issue_code: + errors.append(f"{criterion_label}: issue_code is required") + if criterion.dimension == "authority": + errors.extend( + _authority_errors( + criterion.accepted_authorities, + f"{criterion_label} accepted", + ) + ) + accepted = { + normalize_citation(value) + for value in criterion.accepted_authorities + } + for group in criterion.required_authority_groups: + if not group: + errors.append( + f"{criterion_label}: authority group is empty" + ) + if any(normalize_citation(value) not in accepted for value in group): + errors.append( + f"{criterion_label}: authority group is not accepted" + ) + errors.extend(_document_errors(task, task_dir)) + errors.extend(_review_errors(task.review, label, official)) + return errors + + +def audit_doctrine_overlap( + public_path: str | Path, holdout_path: str | Path +) -> list[str]: + public = load_doctrine_items(public_path) + holdout = load_doctrine_items(holdout_path) + public_families = {item.family_id for item in public} + public_stems = {_normalized_text(item.stem): item.id for item in public} + findings: list[str] = [] + for item in holdout: + if item.family_id in public_families: + findings.append(f"{item.id}: family_id appears in public set") + normalized = _normalized_text(item.stem) + if normalized in public_stems: + findings.append( + f"{item.id}: normalized stem duplicates {public_stems[normalized]}" + ) + return findings diff --git a/tests/test_v4.py b/tests/test_v4.py new file mode 100644 index 0000000..fd547b3 --- /dev/null +++ b/tests/test_v4.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from evidencebench.datasets_v4 import load_doctrine_items, load_matter_tasks +from evidencebench.models_v4 import ( + DoctrineGrounding, + DoctrineResponse, + MatterFinding, + MatterResponse, +) +from evidencebench.runner_v4 import MatterWorkspace +from evidencebench.release_v4 import build_release_manifest +from evidencebench.scoring_v4 import score_doctrine_item, score_matter_task +from evidencebench.statistics_v4 import summarize_suite +from evidencebench.validation_v4 import ( + validate_doctrine_items, + validate_matter_tasks, +) + + +ROOT = Path(__file__).resolve().parents[1] +DOCTRINE = ROOT / "data" / "v4" / "dev" / "doctrine.jsonl" +MATTER = ROOT / "data" / "v4" / "dev" / "matter" + + +class V4ValidationTests(unittest.TestCase): + def test_development_data_validates(self) -> None: + self.assertEqual(validate_doctrine_items(DOCTRINE), []) + self.assertEqual(validate_matter_tasks(MATTER), []) + + def test_draft_data_cannot_be_official(self) -> None: + doctrine_errors = validate_doctrine_items(DOCTRINE, official=True) + matter_errors = validate_matter_tasks(MATTER, official=True) + self.assertTrue(any("must be APPROVED" in item for item in doctrine_errors)) + self.assertTrue(any("must be APPROVED" in item for item in matter_errors)) + + def test_development_manifest_commits_to_data_and_protocol(self) -> None: + manifest = build_release_manifest(DOCTRINE, MATTER, official=False) + self.assertEqual(manifest["release_status"], "development") + self.assertEqual(manifest["doctrine"]["count"], 8) + self.assertEqual(manifest["matter"]["count"], 2) + self.assertEqual(len(manifest["protocol"]["sha256"]), 64) + with self.assertRaises(ValueError): + build_release_manifest(DOCTRINE, MATTER, official=True) + + +class V4ScoringTests(unittest.TestCase): + def test_perfect_doctrine_response_scores_one(self) -> None: + item = load_doctrine_items(DOCTRINE)[3] + response = DoctrineResponse( + item_id=item.id, + ruling="exclude", + issue_codes=["FRE_602_PERSONAL_KNOWLEDGE"], + authorities=["FRE 602"], + grounding=[ + DoctrineGrounding( + "FRE_602_PERSONAL_KNOWLEDGE", ["F1", "F2", "F3"] + ) + ], + confidence=1.0, + ) + score = score_doctrine_item(item, response) + self.assertEqual(score.doctrine_score, 1.0) + self.assertEqual(score.invalid_authorities, []) + + def test_invalid_authority_is_not_silently_dropped(self) -> None: + item = load_doctrine_items(DOCTRINE)[3] + response = DoctrineResponse( + item_id=item.id, + ruling="exclude", + issue_codes=["FRE_602_PERSONAL_KNOWLEDGE"], + authorities=["FRE 602", "Rule banana"], + grounding=[ + DoctrineGrounding( + "FRE_602_PERSONAL_KNOWLEDGE", ["F1", "F2", "F3"] + ) + ], + confidence=1.0, + ) + score = score_doctrine_item(item, response) + self.assertEqual(score.invalid_authorities, ["Rule banana"]) + self.assertEqual(score.authority_precision, 0.5) + self.assertLess(score.doctrine_score, 1.0) + + def test_perfect_matter_response_scores_one(self) -> None: + _, task = load_matter_tasks(MATTER)[0] + response = MatterResponse( + task_id=task.id, + findings=[ + MatterFinding( + issue_code="FRE_901_DISTINCTIVE_CHARACTERISTICS", + disposition="sufficient", + fact_ids=["F2", "F3", "F4"], + record_refs=["record.txt"], + authorities=["FRE 901(a)", "FRE 901(b)(4)"], + ) + ], + deliverables=["findings.json"], + ) + score = score_matter_task(task, response) + self.assertEqual(score.matter_score, 1.0) + self.assertTrue(score.complete_task) + + def test_suite_has_one_overall_score_and_sensitivity(self) -> None: + doctrine_item = load_doctrine_items(DOCTRINE)[3] + doctrine = score_doctrine_item( + doctrine_item, + DoctrineResponse( + doctrine_item.id, + "exclude", + ["FRE_602_PERSONAL_KNOWLEDGE"], + ["FRE 602"], + [ + DoctrineGrounding( + "FRE_602_PERSONAL_KNOWLEDGE", ["F1", "F2", "F3"] + ) + ], + 1.0, + ), + ) + _, matter_task = load_matter_tasks(MATTER)[0] + matter = score_matter_task( + matter_task, + MatterResponse( + matter_task.id, + [ + MatterFinding( + "FRE_901_DISTINCTIVE_CHARACTERISTICS", + "sufficient", + ["F2", "F3", "F4"], + ["record.txt"], + ["FRE 901(a)", "FRE 901(b)(4)"], + ) + ], + ["findings.json"], + ), + ) + summary = summarize_suite([doctrine], [matter]) + self.assertEqual(summary["overall_score"], 1.0) + self.assertEqual(summary["overall_score_100"], 100.0) + self.assertIn("doctrine_40_matter_60", summary["weight_sensitivity"]) + + +class MatterWorkspaceTests(unittest.TestCase): + def test_workspace_separates_read_and_write_roots(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + documents = root / "documents" + outputs = root / "outputs" + documents.mkdir() + (documents / "record.txt").write_text("needle") + workspace = MatterWorkspace(documents, outputs) + result = workspace.execute("search_documents", {"query": "needle"}) + self.assertEqual(result["matches"][0]["path"], "record.txt") + workspace.execute( + "write_output", + {"path": "findings.json", "content": '{"findings":[]}'}, + ) + self.assertTrue((outputs / "findings.json").is_file()) + with self.assertRaises(ValueError): + workspace.execute( + "write_output", {"path": "../escape.txt", "content": "no"} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 506d6f0..b605834 100644 --- a/uv.lock +++ b/uv.lock @@ -4,5 +4,5 @@ requires-python = ">=3.12" [[package]] name = "evidencebench" -version = "1.0.0" +version = "4.0.0" source = { editable = "." } From 906ef1d718bf434fc5ffbd28274c3f8a3607ab9a Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 13:12:00 -0700 Subject: [PATCH 2/8] Harden EvidenceBench v4 corpus protocol --- BENCHMARK_CARD_V4.md | 7 +- METHODOLOGY_V4.md | 47 +- README.md | 12 +- V4_RELEASE_GATES.md | 6 + data/authority-corpus-v4.json | 1530 ++++++++++++++++++++++++++ schemas/v4/doctrine-item.schema.json | 19 + schemas/v4/matter-task.schema.json | 93 +- scripts/build_authority_corpus_v4.py | 159 +++ src/evidencebench/citations.py | 26 + src/evidencebench/cli.py | 9 +- src/evidencebench/models_v4.py | 100 +- src/evidencebench/runner_v4.py | 91 +- src/evidencebench/scoring_v4.py | 253 ++++- src/evidencebench/statistics_v4.py | 95 +- src/evidencebench/validation_v4.py | 108 +- tests/test_v4.py | 186 +++- 16 files changed, 2679 insertions(+), 62 deletions(-) create mode 100644 data/authority-corpus-v4.json create mode 100644 scripts/build_authority_corpus_v4.py diff --git a/BENCHMARK_CARD_V4.md b/BENCHMARK_CARD_V4.md index 4bcc9d8..2d4db06 100644 --- a/BENCHMARK_CARD_V4.md +++ b/BENCHMARK_CARD_V4.md @@ -12,7 +12,7 @@ or evidence that a system can work without attorney review. ## Unit of evaluation -Doctrine units are independently authored fact patterns. Matter units are +Doctrine units are reviewed applied-ruling fact patterns. Matter units are closed-universe workspaces containing instructions, documents, expected deliverables, and atomic criteria. `family_id` is the resampling and leakage boundary. @@ -37,6 +37,11 @@ The single overall score is the equal-weighted mean of Doctrine and Matter. Both track scores and all submetrics remain mandatory, because equal headline scores can conceal materially different capabilities. +Doctrine is family-first and then macro-averaged across 12 domains. Matter +penalizes unsupported extra findings through precision-aware legal, authority, +and factual F1. A separate all-critical-criteria task-resolution rate remains a +strict secondary metric. + ## Quality controls - independent legal review is required for every official item; diff --git a/METHODOLOGY_V4.md b/METHODOLOGY_V4.md index 04f9b76..3a3a12b 100644 --- a/METHODOLOGY_V4.md +++ b/METHODOLOGY_V4.md @@ -51,6 +51,13 @@ required authority groups for which at least one accepted citation is supplied. The benchmark separately records invalid, hallucinated, and real-but-unsupported authorities. +Item scores are not pooled directly. Variants first average within +`family_id`, family scores then average within each of the 12 registered +evidence domains, and the Doctrine track is the unweighted macro-average of +those domain scores. This prevents a four-variant family or a densely sampled +topic from receiving extra headline weight. Confidence intervals resample +families within domains. + ## Matter scoring Expert rubrics are atomic and binary. Scoring dimensions are: @@ -60,12 +67,19 @@ Expert rubrics are atomic and binary. Scoring dimensions are: - factual/record grounding: 15%; and - deliverable completeness: 15%. -The Matter score is the weighted mean of dimension-level pass rates. In -addition, a task-resolution flag passes only if every critical criterion -passes. This preserves the useful strictness of all-pass review without -collapsing the headline metric into a mostly-zero statistic. Criteria marked -`review_only` can support qualitative error analysis but cannot change the -official score. +Legal, authority, and factual dimensions use precision, recall, and F1 against +versioned accepted and required sets. Unsupported extra issues, dispositions, +facts, record references, and authorities reduce precision; missing required +material reduces recall. Deliverable criteria remain deterministic binary +checks of benchmark-observed file bytes and required sections. The Matter score +is the weighted mean of legal F1, authority F1, factual/record F1, and +deliverable pass rate. + +A task-resolution flag passes only if every critical atomic criterion passes +and every scored dimension equals one. This preserves the useful strictness of +all-pass review without collapsing the headline metric into a mostly-zero +statistic. Criteria marked `review_only` can support qualitative error analysis +but cannot change the official score. The evaluator derives deliverable existence from the output filesystem; a model cannot earn credit merely by claiming it wrote a file. @@ -91,11 +105,24 @@ a private, access-controlled repository. Public and holdout data may not share a `family_id`. Exact normalized stem overlap is automatically rejected; release review must additionally test semantic overlap and contamination. -Synthetic matters must use fictional parties and facts. Real cases may be used +Synthetic matters must use fictional parties and facts. Native PDF and DOCX +records have hashed canonical-text companions so extraction-library variance +cannot change the input presented to a model. Real cases may be used only when the source, license, holding, current-law status, and retrieval hash are recorded. Every official item must name an author and at least one independent reviewer and must have status `APPROVED`. +The construction target is 240 Doctrine items and 60 Matter tasks. Doctrine +contains 151 migrated v3 concepts and 89 new coverage items, scored as 168 +families across 12 domains. Its jurisdiction split is 144 federal and 96 state +items. Matter contains five task archetypes in every domain, with 36 federal +and 24 state tasks. The selected state panel is California, New York, Texas, +Florida, Pennsylvania, New Jersey, Illinois, Ohio, Michigan, Georgia, +Washington, and Arizona. +The 89 new items use this panel. Migrated legacy precedent concepts retain +their original jurisdictions, including several states outside the panel, so +historical lineage is not silently rewritten. + ## Required reporting An official result publishes: @@ -111,8 +138,10 @@ An official result publishes: ## Known limitations -EvidenceBench focuses on United States federal evidence doctrine. Development -fixtures are small and synthetic. Deterministic issue codes and rubrics reduce +EvidenceBench focuses on United States federal evidence doctrine and a +population-and-region-selected 12-state panel; it does not claim complete +coverage of every state. Development fixtures are small and synthetic. +Deterministic issue codes and rubrics reduce judge variance but can under-credit unanticipated sound analysis; challenges must be adjudicated and incorporated prospectively in a versioned annotation. OpenRouter can route the same model slug through different upstream providers, diff --git a/README.md b/README.md index 1b30692..bdb3382 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,11 @@ v4 reports a single overall score: `0.50 × Doctrine score + 0.50 × Matter score` It also reports both track scores, submetrics, strict Matter task-resolution -rate, family-clustered 95% bootstrap intervals, and 40/60 versus 60/40 weight -sensitivity. Headline scoring is deterministic; LLM judges cannot change the -official score. +rate, family- and domain-aware 95% bootstrap intervals, and 40/60 versus 60/40 +weight sensitivity. Doctrine variants are averaged within families and then +macro-averaged across 12 domains. Matter uses precision-aware F1 so unsupported +extra issues, authorities, facts, and record references reduce the score. +Headline scoring is deterministic; LLM judges cannot change the official score. See [METHODOLOGY_V4.md](METHODOLOGY_V4.md) and [BENCHMARK_CARD_V4.md](BENCHMARK_CARD_V4.md). The design rationale and the @@ -52,6 +54,10 @@ annotations, and item-level results are not public. Once approved, published manifests will commit to the holdout with a SHA-256 hash, counts, categories, protocol hash, and aggregate metrics. +The private v4 construction target is 240 Doctrine candidates and 60 Matter +tasks. These remain a candidate pool—not an official holdout—until every item +passes the review and freeze gates. + ## v4 commands ```bash diff --git a/V4_RELEASE_GATES.md b/V4_RELEASE_GATES.md index cc0f282..25ff74d 100644 --- a/V4_RELEASE_GATES.md +++ b/V4_RELEASE_GATES.md @@ -19,9 +19,15 @@ No result is “official EvidenceBench v4” until all gates pass. invalid citations, fabricated citations, refusals, and partial failures. - [ ] A frozen scoring version and prompt version are recorded. - [ ] Family-clustered confidence intervals are reported. +- [ ] Doctrine family-first and 12-domain macro-averaging are independently + recomputed from released aggregate records. +- [ ] Precision attacks with extra issues, citations, facts, and record + references are covered by gold-response tests. - [ ] At least three runs per system are completed. - [ ] Track-weight sensitivity is reported. - [ ] Human agreement and adjudication statistics are published. +- [ ] All PDF/DOCX canonical companions match the reviewed native documents, + and a batch render audit finds no truncation or malformed pages. ## Execution gate diff --git a/data/authority-corpus-v4.json b/data/authority-corpus-v4.json new file mode 100644 index 0000000..867ad86 --- /dev/null +++ b/data/authority-corpus-v4.json @@ -0,0 +1,1530 @@ +{ + "authorities": [ + { + "aliases": [], + "canonical": "FRE 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "FRE 502", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "federal", + "source_url": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf" + }, + { + "aliases": [ + "Cal. Evid. Code Section 405", + "Cal Evid Code \u00a7 405" + ], + "canonical": "Cal. Evid. Code \u00a7 405", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 352", + "Cal Evid Code \u00a7 352" + ], + "canonical": "Cal. Evid. Code \u00a7 352", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 1101", + "Cal Evid Code \u00a7 1101" + ], + "canonical": "Cal. Evid. Code \u00a7 1101", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 1151", + "Cal Evid Code \u00a7 1151" + ], + "canonical": "Cal. Evid. Code \u00a7 1151", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 765", + "Cal Evid Code \u00a7 765" + ], + "canonical": "Cal. Evid. Code \u00a7 765", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 780", + "Cal Evid Code \u00a7 780" + ], + "canonical": "Cal. Evid. Code \u00a7 780", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 801", + "Cal Evid Code \u00a7 801" + ], + "canonical": "Cal. Evid. Code \u00a7 801", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 1200", + "Cal Evid Code \u00a7 1200" + ], + "canonical": "Cal. Evid. Code \u00a7 1200", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 1271", + "Cal Evid Code \u00a7 1271" + ], + "canonical": "Cal. Evid. Code \u00a7 1271", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 1400", + "Cal Evid Code \u00a7 1400" + ], + "canonical": "Cal. Evid. Code \u00a7 1400", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 1521", + "Cal Evid Code \u00a7 1521" + ], + "canonical": "Cal. Evid. Code \u00a7 1521", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [ + "Cal. Evid. Code Section 954", + "Cal Evid Code \u00a7 954" + ], + "canonical": "Cal. Evid. Code \u00a7 954", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "california", + "source_url": "https://leginfo.legislature.ca.gov/faces/codes.xhtml" + }, + { + "aliases": [], + "canonical": "GNYE 1.09", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 4.07", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 4.09", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 4.19", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 6.10", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 6.15", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 7.01", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 8.00", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 8.08", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 9.01", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 10.03", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [], + "canonical": "GNYE 5.03", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_york", + "source_url": "https://www.nycourts.gov/guide-new-york-evidence" + }, + { + "aliases": [ + "Tex R Evid 104" + ], + "canonical": "Tex. R. Evid. 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 403" + ], + "canonical": "Tex. R. Evid. 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 404" + ], + "canonical": "Tex. R. Evid. 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 407" + ], + "canonical": "Tex. R. Evid. 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 611" + ], + "canonical": "Tex. R. Evid. 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 613" + ], + "canonical": "Tex. R. Evid. 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 702" + ], + "canonical": "Tex. R. Evid. 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 801" + ], + "canonical": "Tex. R. Evid. 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 803(6)" + ], + "canonical": "Tex. R. Evid. 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 901" + ], + "canonical": "Tex. R. Evid. 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 1002" + ], + "canonical": "Tex. R. Evid. 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Tex R Evid 503" + ], + "canonical": "Tex. R. Evid. 503", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "texas", + "source_url": "https://txcourts.gov/rules-forms/" + }, + { + "aliases": [ + "Fla. Stat. Section 90.105", + "Fla Stat \u00a7 90.105" + ], + "canonical": "Fla. Stat. \u00a7 90.105", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.403", + "Fla Stat \u00a7 90.403" + ], + "canonical": "Fla. Stat. \u00a7 90.403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.404", + "Fla Stat \u00a7 90.404" + ], + "canonical": "Fla. Stat. \u00a7 90.404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.407", + "Fla Stat \u00a7 90.407" + ], + "canonical": "Fla. Stat. \u00a7 90.407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.612", + "Fla Stat \u00a7 90.612" + ], + "canonical": "Fla. Stat. \u00a7 90.612", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.608", + "Fla Stat \u00a7 90.608" + ], + "canonical": "Fla. Stat. \u00a7 90.608", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.702", + "Fla Stat \u00a7 90.702" + ], + "canonical": "Fla. Stat. \u00a7 90.702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.801", + "Fla Stat \u00a7 90.801" + ], + "canonical": "Fla. Stat. \u00a7 90.801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.803(6)", + "Fla Stat \u00a7 90.803(6)" + ], + "canonical": "Fla. Stat. \u00a7 90.803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.901", + "Fla Stat \u00a7 90.901" + ], + "canonical": "Fla. Stat. \u00a7 90.901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.952", + "Fla Stat \u00a7 90.952" + ], + "canonical": "Fla. Stat. \u00a7 90.952", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Fla. Stat. Section 90.502", + "Fla Stat \u00a7 90.502" + ], + "canonical": "Fla. Stat. \u00a7 90.502", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "florida", + "source_url": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html" + }, + { + "aliases": [ + "Pa R.E 104" + ], + "canonical": "Pa. R.E. 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 403" + ], + "canonical": "Pa. R.E. 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 404" + ], + "canonical": "Pa. R.E. 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 407" + ], + "canonical": "Pa. R.E. 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 611" + ], + "canonical": "Pa. R.E. 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 613" + ], + "canonical": "Pa. R.E. 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 702" + ], + "canonical": "Pa. R.E. 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 801" + ], + "canonical": "Pa. R.E. 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 803(6)" + ], + "canonical": "Pa. R.E. 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 901" + ], + "canonical": "Pa. R.E. 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 1002" + ], + "canonical": "Pa. R.E. 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "Pa R.E 502" + ], + "canonical": "Pa. R.E. 502", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "pennsylvania", + "source_url": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence" + }, + { + "aliases": [ + "N.J.R.E 104" + ], + "canonical": "N.J.R.E. 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 403" + ], + "canonical": "N.J.R.E. 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 404" + ], + "canonical": "N.J.R.E. 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 407" + ], + "canonical": "N.J.R.E. 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 611" + ], + "canonical": "N.J.R.E. 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 613" + ], + "canonical": "N.J.R.E. 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 702" + ], + "canonical": "N.J.R.E. 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 801" + ], + "canonical": "N.J.R.E. 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 803(6)" + ], + "canonical": "N.J.R.E. 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 901" + ], + "canonical": "N.J.R.E. 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 1002" + ], + "canonical": "N.J.R.E. 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "N.J.R.E 500" + ], + "canonical": "N.J.R.E. 500", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "new_jersey", + "source_url": "https://www.njcourts.gov/sites/default/files/evidence1.pdf" + }, + { + "aliases": [ + "Ill R Evid 104" + ], + "canonical": "Ill. R. Evid. 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 403" + ], + "canonical": "Ill. R. Evid. 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 404" + ], + "canonical": "Ill. R. Evid. 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 407" + ], + "canonical": "Ill. R. Evid. 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 611" + ], + "canonical": "Ill. R. Evid. 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 613" + ], + "canonical": "Ill. R. Evid. 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 702" + ], + "canonical": "Ill. R. Evid. 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 801" + ], + "canonical": "Ill. R. Evid. 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 803(6)" + ], + "canonical": "Ill. R. Evid. 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 901" + ], + "canonical": "Ill. R. Evid. 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 1002" + ], + "canonical": "Ill. R. Evid. 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ill R Evid 502" + ], + "canonical": "Ill. R. Evid. 502", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "illinois", + "source_url": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/" + }, + { + "aliases": [ + "Ohio R Evid 104" + ], + "canonical": "Ohio R. Evid. 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 403" + ], + "canonical": "Ohio R. Evid. 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 404" + ], + "canonical": "Ohio R. Evid. 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 407" + ], + "canonical": "Ohio R. Evid. 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 611" + ], + "canonical": "Ohio R. Evid. 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 613" + ], + "canonical": "Ohio R. Evid. 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 702" + ], + "canonical": "Ohio R. Evid. 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 801" + ], + "canonical": "Ohio R. Evid. 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 803(6)" + ], + "canonical": "Ohio R. Evid. 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 901" + ], + "canonical": "Ohio R. Evid. 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 1002" + ], + "canonical": "Ohio R. Evid. 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [ + "Ohio R Evid 502" + ], + "canonical": "Ohio R. Evid. 502", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "ohio", + "source_url": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [], + "canonical": "MRE 501", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "michigan", + "source_url": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf" + }, + { + "aliases": [ + "O.C.G.A. Section 24-1-104", + "O.C.G.A \u00a7 24-1-104" + ], + "canonical": "O.C.G.A. \u00a7 24-1-104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-4-403", + "O.C.G.A \u00a7 24-4-403" + ], + "canonical": "O.C.G.A. \u00a7 24-4-403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-4-404", + "O.C.G.A \u00a7 24-4-404" + ], + "canonical": "O.C.G.A. \u00a7 24-4-404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-4-407", + "O.C.G.A \u00a7 24-4-407" + ], + "canonical": "O.C.G.A. \u00a7 24-4-407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-6-611", + "O.C.G.A \u00a7 24-6-611" + ], + "canonical": "O.C.G.A. \u00a7 24-6-611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-6-613", + "O.C.G.A \u00a7 24-6-613" + ], + "canonical": "O.C.G.A. \u00a7 24-6-613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-7-702", + "O.C.G.A \u00a7 24-7-702" + ], + "canonical": "O.C.G.A. \u00a7 24-7-702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-8-801", + "O.C.G.A \u00a7 24-8-801" + ], + "canonical": "O.C.G.A. \u00a7 24-8-801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-8-803", + "O.C.G.A \u00a7 24-8-803" + ], + "canonical": "O.C.G.A. \u00a7 24-8-803", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-9-901", + "O.C.G.A \u00a7 24-9-901" + ], + "canonical": "O.C.G.A. \u00a7 24-9-901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-10-1002", + "O.C.G.A \u00a7 24-10-1002" + ], + "canonical": "O.C.G.A. \u00a7 24-10-1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "O.C.G.A. Section 24-5-501", + "O.C.G.A \u00a7 24-5-501" + ], + "canonical": "O.C.G.A. \u00a7 24-5-501", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "georgia", + "source_url": "https://www.legis.ga.gov/" + }, + { + "aliases": [ + "Wash ER 104" + ], + "canonical": "Wash. ER 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 403" + ], + "canonical": "Wash. ER 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 404" + ], + "canonical": "Wash. ER 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 407" + ], + "canonical": "Wash. ER 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 611" + ], + "canonical": "Wash. ER 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 613" + ], + "canonical": "Wash. ER 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 702" + ], + "canonical": "Wash. ER 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 801" + ], + "canonical": "Wash. ER 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 803(6)" + ], + "canonical": "Wash. ER 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 901" + ], + "canonical": "Wash. ER 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 1002" + ], + "canonical": "Wash. ER 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Wash ER 501" + ], + "canonical": "Wash. ER 501", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "washington", + "source_url": "https://www.courts.wa.gov/court_rules/" + }, + { + "aliases": [ + "Ariz R Evid 104" + ], + "canonical": "Ariz. R. Evid. 104", + "domain": "D01_JUDICIAL_ADMINISTRATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 403" + ], + "canonical": "Ariz. R. Evid. 403", + "domain": "D02_RELEVANCE_403", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 404" + ], + "canonical": "Ariz. R. Evid. 404", + "domain": "D03_CHARACTER_PROPENSITY_HABIT", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 407" + ], + "canonical": "Ariz. R. Evid. 407", + "domain": "D04_POLICY_EXCLUSIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 611" + ], + "canonical": "Ariz. R. Evid. 611", + "domain": "D05_WITNESS_EXAMINATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 613" + ], + "canonical": "Ariz. R. Evid. 613", + "domain": "D06_IMPEACHMENT_REHABILITATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 702" + ], + "canonical": "Ariz. R. Evid. 702", + "domain": "D07_OPINION_EXPERTS", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 801" + ], + "canonical": "Ariz. R. Evid. 801", + "domain": "D08_HEARSAY_DEFINITIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 803(6)" + ], + "canonical": "Ariz. R. Evid. 803(6)", + "domain": "D09_HEARSAY_EXCEPTIONS", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 901" + ], + "canonical": "Ariz. R. Evid. 901", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 1002" + ], + "canonical": "Ariz. R. Evid. 1002", + "domain": "D11_CONTENTS_ORIGINALS_SUMMARIES", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + }, + { + "aliases": [ + "Ariz R Evid 502" + ], + "canonical": "Ariz. R. Evid. 502", + "domain": "D12_PRIVILEGE_CONSTITUTIONAL", + "effective_status": "verify_at_freeze", + "jurisdiction": "arizona", + "source_url": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx" + } + ], + "freeze_policy": "current law on final corpus-freeze date", + "schema_version": "4.0" +} diff --git a/schemas/v4/doctrine-item.schema.json b/schemas/v4/doctrine-item.schema.json index 6b5b5df..d72487e 100644 --- a/schemas/v4/doctrine-item.schema.json +++ b/schemas/v4/doctrine-item.schema.json @@ -9,6 +9,7 @@ "track", "id", "family_id", + "domain", "category", "difficulty", "jurisdiction", @@ -24,6 +25,22 @@ "track": {"const": "doctrine"}, "id": {"type": "string", "minLength": 1}, "family_id": {"type": "string", "minLength": 1}, + "domain": { + "enum": [ + "D01_JUDICIAL_ADMINISTRATION", + "D02_RELEVANCE_403", + "D03_CHARACTER_PROPENSITY_HABIT", + "D04_POLICY_EXCLUSIONS", + "D05_WITNESS_EXAMINATION", + "D06_IMPEACHMENT_REHABILITATION", + "D07_OPINION_EXPERTS", + "D08_HEARSAY_DEFINITIONS", + "D09_HEARSAY_EXCEPTIONS", + "D10_AUTHENTICATION_IDENTIFICATION", + "D11_CONTENTS_ORIGINALS_SUMMARIES", + "D12_PRIVILEGE_CONSTITUTIONAL" + ] + }, "category": {"type": "string", "minLength": 1}, "difficulty": {"enum": ["easy", "medium", "hard"]}, "jurisdiction": {"type": "string", "minLength": 1}, @@ -98,6 +115,8 @@ } }, "corpus_version": {"type": "string", "minLength": 1}, + "legacy_source_id": {"type": ["string", "null"]}, + "coverage_cell": {"type": "string", "minLength": 1}, "review": {"$ref": "#/$defs/review"}, "dimensions": { "type": "object", diff --git a/schemas/v4/matter-task.schema.json b/schemas/v4/matter-task.schema.json index b107003..1431561 100644 --- a/schemas/v4/matter-task.schema.json +++ b/schemas/v4/matter-task.schema.json @@ -9,6 +9,8 @@ "track", "id", "family_id", + "domain", + "jurisdiction", "title", "task_type", "instructions", @@ -23,6 +25,23 @@ "track": {"const": "matter"}, "id": {"type": "string", "minLength": 1}, "family_id": {"type": "string", "minLength": 1}, + "domain": { + "enum": [ + "D01_JUDICIAL_ADMINISTRATION", + "D02_RELEVANCE_403", + "D03_CHARACTER_PROPENSITY_HABIT", + "D04_POLICY_EXCLUSIONS", + "D05_WITNESS_EXAMINATION", + "D06_IMPEACHMENT_REHABILITATION", + "D07_OPINION_EXPERTS", + "D08_HEARSAY_DEFINITIONS", + "D09_HEARSAY_EXCEPTIONS", + "D10_AUTHENTICATION_IDENTIFICATION", + "D11_CONTENTS_ORIGINALS_SUMMARIES", + "D12_PRIVILEGE_CONSTITUTIONAL" + ] + }, + "jurisdiction": {"type": "string", "minLength": 1}, "title": {"type": "string", "minLength": 1}, "task_type": {"type": "string", "minLength": 1}, "instructions": {"type": "string", "minLength": 1}, @@ -35,7 +54,12 @@ "required": ["path", "sha256"], "properties": { "path": {"type": "string", "minLength": 1}, - "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "canonical_text_path": {"type": ["string", "null"]}, + "canonical_text_sha256": { + "type": ["string", "null"], + "pattern": "^[a-f0-9]{64}$" + } } } }, @@ -81,12 +105,79 @@ "items": {"type": "string"} }, "deliverable": {"type": ["string", "null"]}, + "min_bytes": {"type": "integer", "minimum": 1}, + "required_sections": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, "critical": {"type": "boolean"}, "review_only": {"type": "boolean"} } } }, + "gold_findings": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "issue_code", + "accepted_dispositions", + "required_fact_ids", + "accepted_fact_ids", + "required_record_refs", + "accepted_record_refs", + "required_authority_groups", + "accepted_authorities" + ], + "properties": { + "issue_code": {"type": "string", "minLength": 1}, + "accepted_dispositions": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "required_fact_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "accepted_fact_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "required_record_refs": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "accepted_record_refs": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "required_authority_groups": { + "type": "array", + "items": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + } + }, + "accepted_authorities": { + "type": "array", + "items": {"type": "string", "minLength": 1} + } + } + } + }, "corpus_version": {"type": "string", "minLength": 1}, + "legacy_source_id": {"type": ["string", "null"]}, + "coverage_cell": {"type": "string", "minLength": 1}, "review": { "type": "object", "additionalProperties": false, diff --git a/scripts/build_authority_corpus_v4.py b/scripts/build_authority_corpus_v4.py new file mode 100644 index 0000000..6bd6e1d --- /dev/null +++ b/scripts/build_authority_corpus_v4.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + +DOMAINS = [ + "D01_JUDICIAL_ADMINISTRATION", + "D02_RELEVANCE_403", + "D03_CHARACTER_PROPENSITY_HABIT", + "D04_POLICY_EXCLUSIONS", + "D05_WITNESS_EXAMINATION", + "D06_IMPEACHMENT_REHABILITATION", + "D07_OPINION_EXPERTS", + "D08_HEARSAY_DEFINITIONS", + "D09_HEARSAY_EXCEPTIONS", + "D10_AUTHENTICATION_IDENTIFICATION", + "D11_CONTENTS_ORIGINALS_SUMMARIES", + "D12_PRIVILEGE_CONSTITUTIONAL", +] + +SOURCES = { + "federal": "https://www.uscourts.gov/sites/default/files/document/federal-rules-of-evidence.pdf", + "california": "https://leginfo.legislature.ca.gov/faces/codes.xhtml", + "new_york": "https://www.nycourts.gov/guide-new-york-evidence", + "texas": "https://txcourts.gov/rules-forms/", + "florida": "https://www.leg.state.fl.us/statutes/index.cfm?App_mode=Display_Statute&URL=0000-0099/0090/0090ContentsIndex.html", + "pennsylvania": "https://www.pacourts.us/courts/supreme-court/committees/rules-committees/committee-on-rules-of-evidence", + "new_jersey": "https://www.njcourts.gov/sites/default/files/evidence1.pdf", + "illinois": "https://www.illinoiscourts.gov/courts-supreme-court-illinois-rules-of-evidence/", + "ohio": "https://www.supremecourt.ohio.gov/docs/LegalResources/Rules/evidence/evidence.pdf", + "michigan": "https://www.courts.michigan.gov/siteassets/rules-instructions-administrative-orders/rules-of-evidence/michigan-rules-of-evidence.pdf", + "georgia": "https://www.legis.ga.gov/", + "washington": "https://www.courts.wa.gov/court_rules/", + "arizona": "https://www.azcourts.gov/rules/CurrentArizonaRules.aspx", +} + +STANDARD_RULES = ["104", "403", "404", "407", "611", "613", "702", "801", "803(6)", "901", "1002", "502"] +PREFIXES = { + "federal": "FRE", + "texas": "Tex. R. Evid.", + "pennsylvania": "Pa. R.E.", + "new_jersey": "N.J.R.E.", + "illinois": "Ill. R. Evid.", + "ohio": "Ohio R. Evid.", + "michigan": "MRE", + "washington": "Wash. ER", + "arizona": "Ariz. R. Evid.", +} + +SPECIAL = { + "california": [ + "Cal. Evid. Code § 405", + "Cal. Evid. Code § 352", + "Cal. Evid. Code § 1101", + "Cal. Evid. Code § 1151", + "Cal. Evid. Code § 765", + "Cal. Evid. Code § 780", + "Cal. Evid. Code § 801", + "Cal. Evid. Code § 1200", + "Cal. Evid. Code § 1271", + "Cal. Evid. Code § 1400", + "Cal. Evid. Code § 1521", + "Cal. Evid. Code § 954", + ], + "new_york": [ + "GNYE 1.09", + "GNYE 4.07", + "GNYE 4.09", + "GNYE 4.19", + "GNYE 6.10", + "GNYE 6.15", + "GNYE 7.01", + "GNYE 8.00", + "GNYE 8.08", + "GNYE 9.01", + "GNYE 10.03", + "GNYE 5.03", + ], + "florida": [ + "Fla. Stat. § 90.105", + "Fla. Stat. § 90.403", + "Fla. Stat. § 90.404", + "Fla. Stat. § 90.407", + "Fla. Stat. § 90.612", + "Fla. Stat. § 90.608", + "Fla. Stat. § 90.702", + "Fla. Stat. § 90.801", + "Fla. Stat. § 90.803(6)", + "Fla. Stat. § 90.901", + "Fla. Stat. § 90.952", + "Fla. Stat. § 90.502", + ], + "georgia": [ + "O.C.G.A. § 24-1-104", + "O.C.G.A. § 24-4-403", + "O.C.G.A. § 24-4-404", + "O.C.G.A. § 24-4-407", + "O.C.G.A. § 24-6-611", + "O.C.G.A. § 24-6-613", + "O.C.G.A. § 24-7-702", + "O.C.G.A. § 24-8-801", + "O.C.G.A. § 24-8-803", + "O.C.G.A. § 24-9-901", + "O.C.G.A. § 24-10-1002", + "O.C.G.A. § 24-5-501", + ], +} + +# Privilege rules are not numbered uniformly. Keep these explicit instead of +# implying a false cross-jurisdictional parallel to FRE 502. +PRIVILEGE_OVERRIDES = { + "texas": "Tex. R. Evid. 503", + "new_jersey": "N.J.R.E. 500", + "michigan": "MRE 501", + "washington": "Wash. ER 501", +} + + +def _aliases(canonical: str) -> list[str]: + aliases = [canonical.replace("§", "Section")] + aliases.append(canonical.replace(". ", " ").replace("R. Evid.", "Rules of Evidence")) + return list(dict.fromkeys(value for value in aliases if value != canonical)) + + +def main() -> None: + entries = [] + for jurisdiction, source_url in SOURCES.items(): + if jurisdiction in SPECIAL: + citations = SPECIAL[jurisdiction] + else: + prefix = PREFIXES[jurisdiction] + citations = [f"{prefix} {rule}" for rule in STANDARD_RULES] + if jurisdiction in PRIVILEGE_OVERRIDES: + citations[-1] = PRIVILEGE_OVERRIDES[jurisdiction] + for domain, canonical in zip(DOMAINS, citations, strict=True): + entries.append( + { + "canonical": canonical, + "aliases": _aliases(canonical), + "jurisdiction": jurisdiction, + "domain": domain, + "source_url": source_url, + "effective_status": "verify_at_freeze", + } + ) + output = { + "schema_version": "4.0", + "freeze_policy": "current law on final corpus-freeze date", + "authorities": entries, + } + destination = ROOT / "data" / "authority-corpus-v4.json" + destination.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/src/evidencebench/citations.py b/src/evidencebench/citations.py index d2b1f76..eab9630 100644 --- a/src/evidencebench/citations.py +++ b/src/evidencebench/citations.py @@ -42,6 +42,10 @@ def _case_index_path() -> Path: return v3 if v3.exists() else root / "case-authorities-v2.json" +def _v4_authority_path() -> Path: + return Path(__file__).resolve().parents[2] / "data" / "authority-corpus-v4.json" + + def load_rule_index() -> dict[str, list[str]]: return json.loads(_index_path().read_text())["rules"] @@ -50,8 +54,27 @@ def load_case_index() -> set[str]: return set(json.loads(_case_index_path().read_text())["citations"]) +def load_v4_authority_index() -> tuple[dict[str, str], set[str]]: + path = _v4_authority_path() + if not path.exists(): + return {}, set() + entries = json.loads(path.read_text())["authorities"] + aliases: dict[str, str] = {} + canonical: set[str] = set() + for entry in entries: + canonical_value = entry["canonical"] + canonical.add(canonical_value) + for value in [canonical_value, *entry.get("aliases", [])]: + aliases[re.sub(r"\s+", " ", value.strip()).casefold()] = canonical_value + return aliases, canonical + + def normalize_citation(value: str) -> str | None: stripped = value.strip() + v4_aliases, _ = load_v4_authority_index() + registered = v4_aliases.get(re.sub(r"\s+", " ", stripped).casefold()) + if registered: + return registered if re.search(r"(?:FRE|Fed(?:eral)?\.?\s*R(?:ule)?\.?\s*Evid\.?|Rule)\s*\d", stripped, re.I): match = _CITATION.search(stripped) if match: @@ -77,6 +100,9 @@ def citation_exists(citation: str, index: dict[str, list[str]] | None = None) -> normalized = normalize_citation(citation) if not normalized: return False + _, v4_canonical = load_v4_authority_index() + if normalized in v4_canonical: + return True if not normalized.startswith("FRE "): return normalized in load_case_index() index = index or load_rule_index() diff --git a/src/evidencebench/cli.py b/src/evidencebench/cli.py index 0c23c65..6559821 100644 --- a/src/evidencebench/cli.py +++ b/src/evidencebench/cli.py @@ -73,9 +73,13 @@ def _write_or_print(payload: dict, output: str | None) -> None: def _validate_v4(args: argparse.Namespace) -> int: if args.track == "doctrine": - errors = validate_doctrine_items(args.input, official=args.official) + errors = validate_doctrine_items( + args.input, official=args.official, candidate=args.candidate + ) else: - errors = validate_matter_tasks(args.input, official=args.official) + errors = validate_matter_tasks( + args.input, official=args.official, candidate=args.candidate + ) if errors: print("\n".join(errors)) return 1 @@ -212,6 +216,7 @@ def main() -> int: validate_v4.add_argument("--track", choices=("doctrine", "matter"), required=True) validate_v4.add_argument("--input", required=True) validate_v4.add_argument("--official", action="store_true") + validate_v4.add_argument("--candidate", action="store_true") validate_v4.set_defaults(handler=_validate_v4) score_v4 = subparsers.add_parser("score-v4") score_v4.add_argument("--track", choices=("doctrine", "matter"), required=True) diff --git a/src/evidencebench/models_v4.py b/src/evidencebench/models_v4.py index 0d6f9db..6133c90 100644 --- a/src/evidencebench/models_v4.py +++ b/src/evidencebench/models_v4.py @@ -87,6 +87,9 @@ class DoctrineItem: gold: DoctrineGold corpus_version: str review: ReviewRecord + domain: str = "" + legacy_source_id: str | None = None + coverage_cell: str | None = None dimensions: dict[str, str] = field(default_factory=dict) @classmethod @@ -105,6 +108,11 @@ def from_dict(cls, payload: dict[str, Any]) -> "DoctrineItem": gold=DoctrineGold.from_dict(payload["gold"]), corpus_version=payload["corpus_version"], review=ReviewRecord.from_dict(payload["review"]), + domain=payload.get( + "domain", payload.get("dimensions", {}).get("domain", payload["category"]) + ), + legacy_source_id=payload.get("legacy_source_id"), + coverage_cell=payload.get("coverage_cell"), dimensions=dict(payload.get("dimensions", {})), ) @@ -157,6 +165,7 @@ def from_dict(cls, payload: dict[str, Any]) -> "DoctrineResponse": class DoctrineItemScore: item_id: str family_id: str + domain: str outcome_accuracy: float issue_precision: float issue_recall: float @@ -179,10 +188,45 @@ class DoctrineItemScore: class MatterDocument: path: str sha256: str + canonical_text_path: str | None = None + canonical_text_sha256: str | None = None @classmethod def from_dict(cls, payload: dict[str, Any]) -> "MatterDocument": - return cls(path=payload["path"], sha256=payload["sha256"]) + return cls( + path=payload["path"], + sha256=payload["sha256"], + canonical_text_path=payload.get("canonical_text_path"), + canonical_text_sha256=payload.get("canonical_text_sha256"), + ) + + +@dataclass(frozen=True) +class MatterGoldFinding: + issue_code: str + accepted_dispositions: list[str] + required_fact_ids: list[str] + accepted_fact_ids: list[str] + required_record_refs: list[str] + accepted_record_refs: list[str] + required_authority_groups: list[list[str]] + accepted_authorities: list[str] + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "MatterGoldFinding": + return cls( + issue_code=payload["issue_code"], + accepted_dispositions=list(payload["accepted_dispositions"]), + required_fact_ids=list(payload.get("required_fact_ids", [])), + accepted_fact_ids=list(payload.get("accepted_fact_ids", [])), + required_record_refs=list(payload.get("required_record_refs", [])), + accepted_record_refs=list(payload.get("accepted_record_refs", [])), + required_authority_groups=[ + list(group) + for group in payload.get("required_authority_groups", []) + ], + accepted_authorities=list(payload.get("accepted_authorities", [])), + ) @dataclass(frozen=True) @@ -197,6 +241,8 @@ class MatterCriterion: required_authority_groups: list[list[str]] = field(default_factory=list) accepted_authorities: list[str] = field(default_factory=list) deliverable: str | None = None + min_bytes: int = 1 + required_sections: list[str] = field(default_factory=list) critical: bool = True review_only: bool = False @@ -216,6 +262,8 @@ def from_dict(cls, payload: dict[str, Any]) -> "MatterCriterion": ], accepted_authorities=list(payload.get("accepted_authorities", [])), deliverable=payload.get("deliverable"), + min_bytes=payload.get("min_bytes", 1), + required_sections=list(payload.get("required_sections", [])), critical=payload.get("critical", True), review_only=payload.get("review_only", False), ) @@ -235,6 +283,11 @@ class MatterTask: criteria: list[MatterCriterion] corpus_version: str review: ReviewRecord + domain: str = "" + jurisdiction: str = "federal" + gold_findings: list[MatterGoldFinding] = field(default_factory=list) + legacy_source_id: str | None = None + coverage_cell: str | None = None dimensions: dict[str, str] = field(default_factory=dict) @classmethod @@ -256,6 +309,20 @@ def from_dict(cls, payload: dict[str, Any]) -> "MatterTask": ], corpus_version=payload["corpus_version"], review=ReviewRecord.from_dict(payload["review"]), + domain=payload.get( + "domain", + payload.get("dimensions", {}).get("domain", payload["task_type"]), + ), + jurisdiction=payload.get( + "jurisdiction", + payload.get("dimensions", {}).get("jurisdiction", "federal"), + ), + gold_findings=[ + MatterGoldFinding.from_dict(item) + for item in payload.get("gold_findings", []) + ], + legacy_source_id=payload.get("legacy_source_id"), + coverage_cell=payload.get("coverage_cell"), dimensions=dict(payload.get("dimensions", {})), ) @@ -290,6 +357,9 @@ class MatterResponse: findings: list[MatterFinding] deliverables: list[str] status: str = "ok" + deliverable_metadata: list["MatterDeliverableMetadata"] = field( + default_factory=list + ) @classmethod def from_dict(cls, payload: dict[str, Any]) -> "MatterResponse": @@ -300,10 +370,31 @@ def from_dict(cls, payload: dict[str, Any]) -> "MatterResponse": for item in payload.get("findings", []) ], deliverables=list(payload.get("deliverables", [])), + deliverable_metadata=[ + MatterDeliverableMetadata.from_dict(item) + for item in payload.get("deliverable_metadata", []) + ], status=payload.get("status", "ok"), ) +@dataclass(frozen=True) +class MatterDeliverableMetadata: + path: str + bytes: int + sha256: str + sections: list[str] = field(default_factory=list) + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "MatterDeliverableMetadata": + return cls( + path=payload["path"], + bytes=int(payload["bytes"]), + sha256=payload["sha256"], + sections=list(payload.get("sections", [])), + ) + + @dataclass(frozen=True) class MatterCriterionScore: criterion_id: str @@ -317,9 +408,16 @@ class MatterCriterionScore: class MatterTaskScore: task_id: str family_id: str + domain: str legal_criteria_rate: float + legal_precision: float + legal_recall: float authority_grounding_rate: float + authority_precision: float + authority_recall: float factual_accuracy_rate: float + factual_precision: float + factual_recall: float deliverable_completeness_rate: float matter_score: float complete_task: bool diff --git a/src/evidencebench/runner_v4.py b/src/evidencebench/runner_v4.py index 5fc784f..5cae155 100644 --- a/src/evidencebench/runner_v4.py +++ b/src/evidencebench/runner_v4.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import hashlib import os import re import socket @@ -109,13 +110,18 @@ def _json_content(message: dict) -> dict: def doctrine_prompt(item) -> str: facts = "\n".join(f"{fact.id}: {fact.text}" for fact in item.facts) rulings = ", ".join(item.allowed_rulings) + governing_law = ( + "the Federal Rules of Evidence" + if item.jurisdiction == "federal" + else f"the controlling evidence law of {item.jurisdiction.replace('_', ' ').title()}" + ) return f"""You are completing the EvidenceBench v4 Doctrine track. -This is closed-book. Do not browse or use tools. Apply the Federal Rules of -Evidence and return one JSON object only. Use only the supplied fact IDs. +This is closed-book. Do not browse or use tools. Apply {governing_law} and +return one JSON object only. Use only the supplied fact IDs. Required shape: {{"ruling":"admit|exclude|limit|defer","issue_codes":["RULE_..."], -"authorities":["FRE 123(a)"],"grounding":[{{"issue_code":"RULE_...", +"authorities":["controlling rule or case citation"],"grounding":[{{"issue_code":"RULE_...", "fact_ids":["F1"]}}],"confidence":0.0,"explanation":"brief analysis"}} Allowed rulings: {rulings} @@ -284,13 +290,24 @@ def _extract_document(path: Path) -> str: class MatterWorkspace: - def __init__(self, document_root: Path, output_root: Path) -> None: + def __init__( + self, + document_root: Path, + output_root: Path, + *, + document_paths: list[str] | None = None, + canonical_paths: dict[str, str] | None = None, + ) -> None: self.document_root = document_root.resolve() self.output_root = output_root.resolve() + self.document_paths = document_paths + self.canonical_paths = canonical_paths or {} self.output_root.mkdir(parents=True, exist_ok=True) def execute(self, name: str, arguments: dict) -> dict: if name == "list_documents": + if self.document_paths is not None: + return {"documents": sorted(self.document_paths)} return { "documents": [ str(path.relative_to(self.document_root)) @@ -299,16 +316,36 @@ def execute(self, name: str, arguments: dict) -> dict: ] } if name == "read_document": - path = _safe_path(self.document_root, arguments["path"]) - return {"path": arguments["path"], "content": _extract_document(path)} + requested = arguments["path"] + if ( + self.document_paths is not None + and requested not in self.document_paths + ): + raise ValueError("document is not declared by the task") + path = _safe_path(self.document_root, requested) + canonical_path = self.canonical_paths.get(requested) + content = ( + _safe_path(self.document_root, canonical_path).read_text() + if canonical_path + else _extract_document(path) + ) + return {"path": requested, "content": content} if name == "search_documents": query = arguments["query"].casefold() matches = [] for path in sorted(self.document_root.rglob("*")): if not path.is_file(): continue + relative = str(path.relative_to(self.document_root)) + if self.document_paths is not None and relative not in self.document_paths: + continue try: - text = _extract_document(path) + canonical_path = self.canonical_paths.get(relative) + text = ( + _safe_path(self.document_root, canonical_path).read_text() + if canonical_path + else _extract_document(path) + ) except (ValueError, OSError, subprocess.SubprocessError): continue for number, line in enumerate(text.splitlines(), 1): @@ -344,7 +381,7 @@ def matter_prompt(task: MatterTask) -> str: You must write findings.json with this shape: {{"findings":[{{"issue_code":"...","disposition":"...", "fact_ids":["..."],"record_refs":["document:line-or-section"], -"authorities":["FRE 123(a)"],"explanation":"..."}}]}} +"authorities":["controlling rule or case citation"],"explanation":"..."}}]}} Use write_output for every deliverable. When complete, respond briefly. """ @@ -355,7 +392,16 @@ def _run_matter_task( task: MatterTask, output_root: Path, ) -> tuple[dict, list[dict], dict]: - workspace = MatterWorkspace(task_dir / "documents", output_root) + workspace = MatterWorkspace( + task_dir / "documents", + output_root, + document_paths=[document.path for document in task.documents], + canonical_paths={ + document.path: document.canonical_text_path + for document in task.documents + if document.canonical_text_path + }, + ) messages: list[dict] = [{"role": "user", "content": matter_prompt(task)}] transcript: list[dict] = list(messages) total_usage: dict[str, int] = {} @@ -408,10 +454,37 @@ def _run_matter_task( for path in sorted(output_root.rglob("*")) if path.is_file() ] + deliverable_metadata = [] + for relative in actual_deliverables: + path = output_root / relative + content = path.read_text(errors="replace") + sections = [] + if path.suffix.casefold() == ".md": + sections = [ + match.group(1).strip() + for line in content.splitlines() + if (match := re.match(r"^#{1,6}\s+(.+?)\s*$", line)) + ] + elif path.suffix.casefold() == ".json": + try: + parsed_content = json.loads(content) + if isinstance(parsed_content, dict): + sections = list(parsed_content) + except json.JSONDecodeError: + pass + deliverable_metadata.append( + { + "path": relative, + "bytes": path.stat().st_size, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "sections": sections, + } + ) response = { "task_id": task.id, "findings": parsed["findings"], "deliverables": actual_deliverables, + "deliverable_metadata": deliverable_metadata, "status": "ok", } return response, transcript, total_usage diff --git a/src/evidencebench/scoring_v4.py b/src/evidencebench/scoring_v4.py index d7cdc16..ec747f7 100644 --- a/src/evidencebench/scoring_v4.py +++ b/src/evidencebench/scoring_v4.py @@ -126,6 +126,7 @@ def score_doctrine_item( return DoctrineItemScore( item_id=item.id, family_id=item.family_id, + domain=item.domain, outcome_accuracy=0.0, issue_precision=0.0, issue_recall=0.0, @@ -193,6 +194,7 @@ def score_doctrine_item( return DoctrineItemScore( item_id=item.id, family_id=item.family_id, + domain=item.domain, outcome_accuracy=outcome, issue_precision=issue_precision, issue_recall=issue_recall, @@ -237,9 +239,26 @@ def _criterion_passes( if criterion.review_only: return False if criterion.dimension == "deliverable": - return bool( - criterion.deliverable - and criterion.deliverable in set(response.deliverables) + if not criterion.deliverable: + return False + metadata = { + item.path: item for item in response.deliverable_metadata + }.get(criterion.deliverable) + if metadata is None: + return ( + criterion.min_bytes <= 1 + and not criterion.required_sections + and criterion.deliverable in set(response.deliverables) + ) + required_sections = { + section.strip().casefold() for section in criterion.required_sections + } + actual_sections = { + section.strip().casefold() for section in metadata.sections + } + return ( + metadata.bytes >= criterion.min_bytes + and required_sections.issubset(actual_sections) ) finding = _matching_finding(criterion, response.findings) @@ -272,6 +291,171 @@ def _criterion_passes( raise ValueError(f"unsupported matter criterion dimension: {criterion.dimension}") +def _legal_f1(task: MatterTask, response: MatterResponse) -> tuple[float, float, float]: + if not task.gold_findings: + return 0.0, 0.0, 0.0 + gold_by_issue = { + finding.issue_code.strip().upper(): { + disposition.strip().casefold() + for disposition in finding.accepted_dispositions + } + for finding in task.gold_findings + } + predicted = { + ( + finding.issue_code.strip().upper(), + (finding.disposition or "").strip().casefold(), + ) + for finding in response.findings + if finding.issue_code.strip() + } + supported = { + pair + for pair in predicted + if pair[0] in gold_by_issue and pair[1] in gold_by_issue[pair[0]] + } + precision = len(supported) / len(predicted) if predicted else 0.0 + recalled_issues = {issue for issue, _ in supported} + recall = len(recalled_issues) / len(gold_by_issue) if gold_by_issue else 1.0 + f1 = ( + 0.0 + if precision + recall == 0 + else 2 * precision * recall / (precision + recall) + ) + return precision, recall, f1 + + +def _matter_authority_f1( + task: MatterTask, response: MatterResponse +) -> tuple[float, float, float, list[str], list[str]]: + gold_by_issue = { + finding.issue_code.strip().upper(): finding + for finding in task.gold_findings + } + submitted_count = 0 + supported_count = 0 + invalid: list[str] = [] + hallucinated: list[str] = [] + supported_by_issue: dict[str, set[str]] = defaultdict(set) + for finding in response.findings: + issue = finding.issue_code.strip().upper() + gold = gold_by_issue.get(issue) + accepted = { + normalized + for value in (gold.accepted_authorities if gold else []) + if (normalized := normalize_citation(value)) is not None + } + seen: set[str] = set() + for raw_value in finding.authorities: + raw = raw_value.strip() + if not raw: + continue + normalized = normalize_citation(raw) + key = normalized or f"INVALID::{raw.casefold()}" + if key in seen: + continue + seen.add(key) + submitted_count += 1 + if normalized is None: + invalid.append(raw) + elif normalized in accepted: + supported_count += 1 + supported_by_issue[issue].add(normalized) + elif not citation_exists(normalized): + hallucinated.append(raw) + precision = supported_count / submitted_count if submitted_count else 0.0 + required_total = 0 + required_met = 0 + for issue, gold in gold_by_issue.items(): + for group in gold.required_authority_groups: + normalized_group = { + normalized + for value in group + if (normalized := normalize_citation(value)) is not None + } + required_total += 1 + required_met += bool(normalized_group & supported_by_issue[issue]) + recall = required_met / required_total if required_total else 1.0 + f1 = ( + 0.0 + if precision + recall == 0 + else 2 * precision * recall / (precision + recall) + ) + return ( + precision, + recall, + f1, + sorted(set(invalid)), + sorted(set(hallucinated)), + ) + + +def _ref_is_accepted(submitted: str, accepted: str) -> bool: + return ( + submitted == accepted + or submitted.startswith(f"{accepted}:") + or submitted.startswith(f"{accepted}#") + ) + + +def _matter_fact_f1( + task: MatterTask, response: MatterResponse +) -> tuple[float, float, float]: + gold_by_issue = { + finding.issue_code.strip().upper(): finding + for finding in task.gold_findings + } + predicted: set[tuple[str, str, str]] = set() + accepted: set[tuple[str, str, str]] = set() + required: set[tuple[str, str, str]] = set() + for issue, gold in gold_by_issue.items(): + accepted.update( + (issue, "fact", value.strip().upper()) + for value in gold.accepted_fact_ids + ) + required.update( + (issue, "fact", value.strip().upper()) + for value in gold.required_fact_ids + ) + accepted.update( + (issue, "ref", value.strip()) for value in gold.accepted_record_refs + ) + required.update( + (issue, "ref", value.strip()) for value in gold.required_record_refs + ) + for finding in response.findings: + issue = finding.issue_code.strip().upper() + predicted.update( + (issue, "fact", value.strip().upper()) + for value in finding.fact_ids + if value.strip() + ) + for submitted in finding.record_refs: + value = submitted.strip() + if not value: + continue + canonical = next( + ( + gold_value + for gold_issue, kind, gold_value in accepted + if gold_issue == issue + and kind == "ref" + and _ref_is_accepted(value, gold_value) + ), + value, + ) + predicted.add((issue, "ref", canonical)) + true_positive = len(predicted & accepted) + precision = true_positive / len(predicted) if predicted else 0.0 + recall = len(predicted & required) / len(required) if required else 1.0 + f1 = ( + 0.0 + if precision + recall == 0 + else 2 * precision * recall / (precision + recall) + ) + return precision, recall, f1 + + def score_matter_task( task: MatterTask, response: MatterResponse ) -> MatterTaskScore: @@ -279,9 +463,16 @@ def score_matter_task( return MatterTaskScore( task_id=task.id, family_id=task.family_id, + domain=task.domain, legal_criteria_rate=0.0, + legal_precision=0.0, + legal_recall=0.0, authority_grounding_rate=0.0, + authority_precision=0.0, + authority_recall=0.0, factual_accuracy_rate=0.0, + factual_precision=0.0, + factual_recall=0.0, deliverable_completeness_rate=0.0, matter_score=0.0, complete_task=False, @@ -310,9 +501,36 @@ def rate(dimension: str) -> float: values = by_dimension[dimension] return sum(score.passed for score in values) / len(values) if values else 0.0 - legal = rate("legal") - authority = rate("authority") - fact = rate("fact") + if task.gold_findings: + legal_precision, legal_recall, legal = _legal_f1(task, response) + ( + authority_precision, + authority_recall, + authority, + invalid, + hallucinated, + ) = _matter_authority_f1(task, response) + fact_precision, fact_recall, fact = _matter_fact_f1(task, response) + else: + legal = rate("legal") + authority = rate("authority") + fact = rate("fact") + legal_precision = legal_recall = legal + authority_precision = authority_recall = authority + fact_precision = fact_recall = fact + all_authorities = [ + authority_value + for finding in response.findings + for authority_value in finding.authorities + ] + accepted = [ + authority_value + for criterion in task.criteria + for authority_value in criterion.accepted_authorities + ] + _, _, _, invalid, hallucinated, _ = _authority_score( + all_authorities, accepted, [] + ) deliverable = rate("deliverable") matter_score = ( MATTER_WEIGHTS["legal"] * legal @@ -325,29 +543,24 @@ def rate(dimension: str) -> float: for score in criterion_scores if score.critical and not score.review_only ] - all_authorities = [ - authority_value - for finding in response.findings - for authority_value in finding.authorities - ] - accepted = [ - authority_value - for criterion in task.criteria - for authority_value in criterion.accepted_authorities - ] - _, _, _, invalid, hallucinated, _ = _authority_score( - all_authorities, accepted, [] - ) return MatterTaskScore( task_id=task.id, family_id=task.family_id, + domain=task.domain, legal_criteria_rate=legal, + legal_precision=legal_precision, + legal_recall=legal_recall, authority_grounding_rate=authority, + authority_precision=authority_precision, + authority_recall=authority_recall, factual_accuracy_rate=fact, + factual_precision=fact_precision, + factual_recall=fact_recall, deliverable_completeness_rate=deliverable, matter_score=matter_score, complete_task=bool(critical_scores) - and all(score.passed for score in critical_scores), + and all(score.passed for score in critical_scores) + and all(value == 1.0 for value in (legal, authority, fact, deliverable)), criteria=criterion_scores, invalid_authorities=invalid, hallucinated_authorities=hallucinated, diff --git a/src/evidencebench/statistics_v4.py b/src/evidencebench/statistics_v4.py index 5618882..d0a6a23 100644 --- a/src/evidencebench/statistics_v4.py +++ b/src/evidencebench/statistics_v4.py @@ -58,11 +58,57 @@ def summarize_doctrine(scores: list[DoctrineItemScore]) -> dict: "grounding_f1", "calibration", ) + families: dict[str, list[DoctrineItemScore]] = defaultdict(list) + for score in scores: + families[score.family_id].append(score) + family_records = [ + { + "family_id": family_id, + "domain": values[0].domain, + **{ + metric_name: mean( + getattr(value, metric_name) for value in values + ) + for metric_name in metric_names + }, + } + for family_id, values in sorted(families.items()) + ] + domains: dict[str, list[dict]] = defaultdict(list) + for record in family_records: + domains[record["domain"]].append(record) + + def domain_macro(metric_name: str) -> float: + return ( + mean( + mean(record[metric_name] for record in records) + for records in domains.values() + ) + if domains + else 0.0 + ) + result = { "n": len(scores), - **{ - name: mean(getattr(score, name) for score in scores) if scores else 0.0 - for name in metric_names + "family_count": len(family_records), + "domain_count": len(domains), + **{name: domain_macro(name) for name in metric_names}, + "item_mean_score": mean(score.doctrine_score for score in scores) + if scores + else 0.0, + "family_mean_score": mean( + record["doctrine_score"] for record in family_records + ) + if family_records + else 0.0, + "by_domain": { + domain: { + "family_count": len(records), + "doctrine_score": mean( + record["doctrine_score"] for record in records + ), + } + for domain, records in sorted(domains.items()) }, "invalid_authority_count": sum( len(score.invalid_authorities) for score in scores @@ -72,8 +118,24 @@ def summarize_doctrine(scores: list[DoctrineItemScore]) -> dict: ), "failure_count": sum(score.status != "ok" for score in scores), } - low, high = cluster_bootstrap_interval( - scores, lambda score: score.family_id, lambda score: score.doctrine_score + generator = random.Random(20260304) + bootstrap_values: list[float] = [] + for _ in range(5000): + bootstrap_values.append( + mean( + mean( + generator.choice(records)["doctrine_score"] + for _ in records + ) + for records in domains.values() + ) + if domains + else 0.0 + ) + bootstrap_values.sort() + low, high = ( + _percentile(bootstrap_values, 0.025), + _percentile(bootstrap_values, 0.975), ) result["score_ci_95"] = [low, high] return result @@ -127,23 +189,32 @@ def summarize_suite( doctrine_groups[score.family_id].append(score) for score in matter_scores: matter_groups[score.family_id].append(score) - doctrine_families = sorted(doctrine_groups) + doctrine_domains: dict[str, list[str]] = defaultdict(list) + for family_id, values in doctrine_groups.items(): + doctrine_domains[values[0].domain].append(family_id) matter_families = sorted(matter_groups) generator = random.Random(20260304) samples: list[float] = [] for _ in range(5000): - sampled_doctrine = [ - item - for _ in doctrine_families - for item in doctrine_groups[generator.choice(doctrine_families)] - ] sampled_matter = [ item for _ in matter_families for item in matter_groups[generator.choice(matter_families)] ] samples.append( - 0.5 * mean(score.doctrine_score for score in sampled_doctrine) + 0.5 + * mean( + mean( + mean( + item.doctrine_score + for item in doctrine_groups[ + generator.choice(family_ids) + ] + ) + for _ in family_ids + ) + for family_ids in doctrine_domains.values() + ) + 0.5 * mean(score.matter_score for score in sampled_matter) ) samples.sort() diff --git a/src/evidencebench/validation_v4.py b/src/evidencebench/validation_v4.py index 2ff5d5e..f469c8f 100644 --- a/src/evidencebench/validation_v4.py +++ b/src/evidencebench/validation_v4.py @@ -14,6 +14,20 @@ REVIEW_STATUSES = {"DRAFT", "IN_REVIEW", "APPROVED"} RULINGS = {"admit", "exclude", "limit", "defer"} MATTER_DIMENSIONS = {"legal", "authority", "fact", "deliverable"} +DOMAINS = { + "D01_JUDICIAL_ADMINISTRATION", + "D02_RELEVANCE_403", + "D03_CHARACTER_PROPENSITY_HABIT", + "D04_POLICY_EXCLUSIONS", + "D05_WITNESS_EXAMINATION", + "D06_IMPEACHMENT_REHABILITATION", + "D07_OPINION_EXPERTS", + "D08_HEARSAY_DEFINITIONS", + "D09_HEARSAY_EXCEPTIONS", + "D10_AUTHENTICATION_IDENTIFICATION", + "D11_CONTENTS_ORIGINALS_SUMMARIES", + "D12_PRIVILEGE_CONSTITUTIONAL", +} def _review_errors(review: ReviewRecord, label: str, official: bool) -> list[str]: @@ -49,7 +63,9 @@ def _normalized_text(value: str) -> str: return re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", value.casefold())).strip() -def validate_doctrine_items(path: str | Path, official: bool = False) -> list[str]: +def validate_doctrine_items( + path: str | Path, official: bool = False, candidate: bool = False +) -> list[str]: errors: list[str] = [] try: items = load_doctrine_items(path) @@ -61,6 +77,7 @@ def validate_doctrine_items(path: str | Path, official: bool = False) -> list[st errors.append(f"duplicate doctrine id: {item_id}") normalized_stems: dict[str, str] = {} + family_domains: dict[str, str] = {} for item in items: label = item.id if item.schema_version != SCHEMA_VERSION: @@ -69,6 +86,14 @@ def validate_doctrine_items(path: str | Path, official: bool = False) -> list[st errors.append(f"{label}: track must be doctrine") if not item.family_id: errors.append(f"{label}: family_id is required") + if official or candidate: + if item.domain not in DOMAINS: + errors.append(f"{label}: domain is not in the v4 registry") + if not item.coverage_cell: + errors.append(f"{label}: coverage_cell is required") + prior_domain = family_domains.setdefault(item.family_id, item.domain) + if prior_domain != item.domain: + errors.append(f"{label}: family spans multiple domains") if item.gold.ruling not in item.allowed_rulings: errors.append(f"{label}: gold ruling is not in allowed_rulings") if set(item.allowed_rulings) - RULINGS: @@ -135,10 +160,38 @@ def _document_errors(task: MatterTask, task_dir: Path) -> list[str]: digest = hashlib.sha256(candidate.read_bytes()).hexdigest() if digest != document.sha256: errors.append(f"{task.id}: sha256 mismatch for {document.path!r}") + if document.canonical_text_path: + canonical = (document_root / document.canonical_text_path).resolve() + try: + canonical.relative_to(document_root) + except ValueError: + errors.append( + f"{task.id}: unsafe canonical text path " + f"{document.canonical_text_path!r}" + ) + continue + if not canonical.is_file(): + errors.append( + f"{task.id}: missing canonical text " + f"{document.canonical_text_path!r}" + ) + elif not document.canonical_text_sha256: + errors.append( + f"{task.id}: canonical text sha256 is required" + ) + elif hashlib.sha256(canonical.read_bytes()).hexdigest() != ( + document.canonical_text_sha256 + ): + errors.append( + f"{task.id}: canonical text sha256 mismatch for " + f"{document.canonical_text_path!r}" + ) return errors -def validate_matter_tasks(path: str | Path, official: bool = False) -> list[str]: +def validate_matter_tasks( + path: str | Path, official: bool = False, candidate: bool = False +) -> list[str]: errors: list[str] = [] root = Path(path) try: @@ -158,6 +211,13 @@ def validate_matter_tasks(path: str | Path, official: bool = False) -> list[str] errors.append(f"{label}: track must be matter") if not task.family_id: errors.append(f"{label}: family_id is required") + if official or candidate: + if task.domain not in DOMAINS: + errors.append(f"{label}: domain is not in the v4 registry") + if not task.coverage_cell: + errors.append(f"{label}: coverage_cell is required") + if not task.gold_findings: + errors.append(f"{label}: gold_findings are required") if not task.documents: errors.append(f"{label}: at least one matter document is required") if not task.deliverables: @@ -186,6 +246,10 @@ def validate_matter_tasks(path: str | Path, official: bool = False) -> list[str] errors.append( f"{criterion_label}: deliverable is not declared by task" ) + if criterion.min_bytes < 1: + errors.append( + f"{criterion_label}: min_bytes must be positive" + ) else: if not criterion.issue_code: errors.append(f"{criterion_label}: issue_code is required") @@ -209,6 +273,46 @@ def validate_matter_tasks(path: str | Path, official: bool = False) -> list[str] errors.append( f"{criterion_label}: authority group is not accepted" ) + gold_issues = [finding.issue_code for finding in task.gold_findings] + if len(gold_issues) != len(set(gold_issues)): + errors.append(f"{label}: gold finding issue codes must be unique") + for finding in task.gold_findings: + gold_label = f"{label}/{finding.issue_code}" + if not finding.accepted_dispositions: + errors.append( + f"{gold_label}: accepted_dispositions cannot be empty" + ) + if not set(finding.required_fact_ids).issubset( + finding.accepted_fact_ids + ): + errors.append( + f"{gold_label}: required facts must be accepted" + ) + if not set(finding.required_record_refs).issubset( + finding.accepted_record_refs + ): + errors.append( + f"{gold_label}: required record refs must be accepted" + ) + accepted = { + normalize_citation(value) + for value in finding.accepted_authorities + } + errors.extend( + _authority_errors( + finding.accepted_authorities, + f"{gold_label} accepted", + ) + ) + for group in finding.required_authority_groups: + if not group: + errors.append(f"{gold_label}: authority group is empty") + if any( + normalize_citation(value) not in accepted for value in group + ): + errors.append( + f"{gold_label}: required authority is not accepted" + ) errors.extend(_document_errors(task, task_dir)) errors.extend(_review_errors(task.review, label, official)) return errors diff --git a/tests/test_v4.py b/tests/test_v4.py index fd547b3..8ad4571 100644 --- a/tests/test_v4.py +++ b/tests/test_v4.py @@ -6,15 +6,18 @@ from evidencebench.datasets_v4 import load_doctrine_items, load_matter_tasks from evidencebench.models_v4 import ( + DoctrineItemScore, DoctrineGrounding, DoctrineResponse, + MatterDeliverableMetadata, MatterFinding, MatterResponse, + MatterTask, ) -from evidencebench.runner_v4 import MatterWorkspace +from evidencebench.runner_v4 import MatterWorkspace, doctrine_prompt from evidencebench.release_v4 import build_release_manifest from evidencebench.scoring_v4 import score_doctrine_item, score_matter_task -from evidencebench.statistics_v4 import summarize_suite +from evidencebench.statistics_v4 import summarize_doctrine, summarize_suite from evidencebench.validation_v4 import ( validate_doctrine_items, validate_matter_tasks, @@ -143,6 +146,141 @@ def test_suite_has_one_overall_score_and_sensitivity(self) -> None: self.assertEqual(summary["overall_score_100"], 100.0) self.assertIn("doctrine_40_matter_60", summary["weight_sensitivity"]) + def test_extra_matter_findings_reduce_precision(self) -> None: + task = MatterTask.from_dict( + { + "schema_version": "4.0", + "track": "matter", + "id": "M-PRECISION", + "family_id": "M-PRECISION", + "domain": "D10_AUTHENTICATION_IDENTIFICATION", + "jurisdiction": "federal", + "title": "Precision test", + "task_type": "admissibility_memo", + "instructions": "Resolve the record.", + "documents": [{"path": "record.txt", "sha256": "0" * 64}], + "deliverables": ["findings.json"], + "criteria": [ + { + "id": "L1", + "dimension": "legal", + "title": "Correct result", + "issue_code": "AUTH", + "expected_disposition": "admit", + }, + { + "id": "A1", + "dimension": "authority", + "title": "Correct authority", + "issue_code": "AUTH", + "required_authority_groups": [["FRE 901(a)"]], + "accepted_authorities": ["FRE 901(a)"], + }, + { + "id": "F1", + "dimension": "fact", + "title": "Correct fact and record", + "issue_code": "AUTH", + "required_fact_ids": ["F1"], + "required_record_refs": ["record.txt"], + }, + { + "id": "D1", + "dimension": "deliverable", + "title": "Structured findings", + "deliverable": "findings.json", + "min_bytes": 10, + "required_sections": ["findings"], + }, + ], + "gold_findings": [ + { + "issue_code": "AUTH", + "accepted_dispositions": ["admit"], + "required_fact_ids": ["F1"], + "accepted_fact_ids": ["F1"], + "required_record_refs": ["record.txt"], + "accepted_record_refs": ["record.txt"], + "required_authority_groups": [["FRE 901(a)"]], + "accepted_authorities": ["FRE 901(a)"], + } + ], + "corpus_version": "test", + "review": {"status": "DRAFT"}, + "coverage_cell": "test", + } + ) + response = MatterResponse( + task_id=task.id, + findings=[ + MatterFinding( + "AUTH", + "admit", + ["F1"], + ["record.txt"], + ["FRE 901(a)"], + ), + MatterFinding( + "UNSUPPORTED", + "exclude", + ["F99"], + ["other.txt"], + ["FRE 403"], + ), + ], + deliverables=["findings.json"], + deliverable_metadata=[ + MatterDeliverableMetadata( + path="findings.json", + bytes=100, + sha256="0" * 64, + sections=["findings"], + ) + ], + ) + score = score_matter_task(task, response) + self.assertAlmostEqual(score.legal_precision, 0.5) + self.assertAlmostEqual(score.authority_precision, 0.5) + self.assertAlmostEqual(score.factual_precision, 0.5) + self.assertAlmostEqual(score.legal_recall, 1.0) + self.assertFalse(score.complete_task) + + def test_doctrine_uses_family_first_domain_macro_average(self) -> None: + def record(item: str, family: str, domain: str, value: float) -> DoctrineItemScore: + return DoctrineItemScore( + item_id=item, + family_id=family, + domain=domain, + outcome_accuracy=value, + issue_precision=value, + issue_recall=value, + issue_f1=value, + authority_precision=value, + authority_recall=value, + authority_f1=value, + grounding_precision=value, + grounding_recall=value, + grounding_f1=value, + calibration=value, + invalid_authorities=[], + hallucinated_authorities=[], + unsupported_authorities=[], + doctrine_score=value, + status="ok", + ) + + summary = summarize_doctrine( + [ + record("A1", "F1", "D01", 1.0), + record("A2", "F1", "D01", 0.0), + record("A3", "F2", "D01", 1.0), + record("B1", "F3", "D02", 0.0), + ] + ) + self.assertEqual(summary["item_mean_score"], 0.5) + self.assertEqual(summary["family_mean_score"], 0.5) + self.assertEqual(summary["doctrine_score"], 0.375) + class MatterWorkspaceTests(unittest.TestCase): def test_workspace_separates_read_and_write_roots(self) -> None: @@ -165,6 +303,50 @@ def test_workspace_separates_read_and_write_roots(self) -> None: "write_output", {"path": "../escape.txt", "content": "no"} ) + def test_workspace_exposes_canonical_text_only_through_declared_document(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + documents = root / "documents" + outputs = root / "outputs" + canonical = documents / "_canonical" + canonical.mkdir(parents=True) + (documents / "record.docx").write_bytes(b"not used") + (canonical / "record.txt").write_text("canonical needle") + workspace = MatterWorkspace( + documents, + outputs, + document_paths=["record.docx"], + canonical_paths={"record.docx": "_canonical/record.txt"}, + ) + self.assertEqual( + workspace.execute("list_documents", {})["documents"], + ["record.docx"], + ) + self.assertEqual( + workspace.execute( + "read_document", {"path": "record.docx"} + )["content"], + "canonical needle", + ) + with self.assertRaises(ValueError): + workspace.execute( + "read_document", {"path": "_canonical/record.txt"} + ) + + +class PromptTests(unittest.TestCase): + def test_state_doctrine_prompt_uses_state_law(self) -> None: + item = load_doctrine_items(DOCTRINE)[0] + state_item = item.__class__.from_dict( + { + **item.to_dict(), + "jurisdiction": "california", + } + ) + prompt = doctrine_prompt(state_item) + self.assertIn("controlling evidence law of California", prompt) + self.assertNotIn("Apply the Federal Rules of Evidence", prompt) + if __name__ == "__main__": unittest.main() From 638489bb3c5bdd0ec6509683b8d08380773142e9 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 13:30:45 -0700 Subject: [PATCH 3/8] Add restart-safe frontier v4 runner --- .../evidencebench-v4-reviewer.yml | 69 +++++ README.md | 3 + REVIEWING_V4.md | 48 ++++ .../claude-opus-5-doctrine.json | 17 ++ .../claude-opus-5-matter.json | 18 ++ .../gemini-3-1-pro-doctrine.json | 17 ++ .../gemini-3-1-pro-matter.json | 18 ++ .../gpt-5-6-sol-doctrine.json | 17 ++ .../frontier-2026-07/gpt-5-6-sol-matter.json | 18 ++ .../frontier-2026-07/grok-4-5-doctrine.json | 17 ++ .../v4/frontier-2026-07/grok-4-5-matter.json | 18 ++ src/evidencebench/runner_v4.py | 254 ++++++++++++++---- tests/test_v4.py | 42 ++- 13 files changed, 509 insertions(+), 47 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/evidencebench-v4-reviewer.yml create mode 100644 REVIEWING_V4.md create mode 100644 models/v4/frontier-2026-07/claude-opus-5-doctrine.json create mode 100644 models/v4/frontier-2026-07/claude-opus-5-matter.json create mode 100644 models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json create mode 100644 models/v4/frontier-2026-07/gemini-3-1-pro-matter.json create mode 100644 models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json create mode 100644 models/v4/frontier-2026-07/gpt-5-6-sol-matter.json create mode 100644 models/v4/frontier-2026-07/grok-4-5-doctrine.json create mode 100644 models/v4/frontier-2026-07/grok-4-5-matter.json diff --git a/.github/ISSUE_TEMPLATE/evidencebench-v4-reviewer.yml b/.github/ISSUE_TEMPLATE/evidencebench-v4-reviewer.yml new file mode 100644 index 0000000..4eb43ef --- /dev/null +++ b/.github/ISSUE_TEMPLATE/evidencebench-v4-reviewer.yml @@ -0,0 +1,69 @@ +name: EvidenceBench v4 professional reviewer +description: Volunteer to independently review one or more sealed EvidenceBench v4 candidates. +title: "[Reviewer application] " +labels: + - reviewer-application +body: + - type: markdown + attributes: + value: | + Thank you for volunteering. EvidenceBench v4 needs identity-verified legal professionals to review sealed evidence-law items before release. + + **Do not post bar numbers, identification documents, client information, or other sensitive personal data in this public issue.** Objection Academy will arrange private verification and assignment after initial triage. + - type: input + id: professional_role + attributes: + label: Professional role + description: For example, litigation attorney, judge, law professor, appellate lawyer, or evidence instructor. + placeholder: Litigation attorney + validations: + required: true + - type: input + id: jurisdictions + attributes: + label: Jurisdictions or evidence-law experience + description: List relevant jurisdictions or describe your evidence-law teaching, research, or practice experience. + placeholder: California and federal courts; trial-practice instructor + validations: + required: true + - type: dropdown + id: capacity + attributes: + label: Review capacity + description: Each assignment is a private Doctrine item or Matter workspace. + options: + - 1–2 candidates + - 3–5 candidates + - 6–10 candidates + - More than 10 candidates + validations: + required: true + - type: checkboxes + id: tracks + attributes: + label: Preferred track + options: + - label: Doctrine — applied evidentiary rulings + - label: Matter — closed-universe files and work product + - type: textarea + id: background + attributes: + label: Relevant background + description: Briefly describe the experience that would help with this review. + validations: + required: true + - type: checkboxes + id: attestations + attributes: + label: Review commitments + options: + - label: I can keep assigned benchmark material confidential. + required: true + - label: I will disclose conflicts and will not use assigned material to train or prompt a model. + required: true + - label: I understand that identity verification will occur privately before access is granted. + required: true + - type: markdown + attributes: + value: | + If you prefer not to open a public issue, email **dylan@examp.com** with the subject “EvidenceBench v4 reviewer.” diff --git a/README.md b/README.md index bdb3382..8eebeb6 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ EvidenceBench v4 must not be presented as an official leaderboard until the legal-review and release gates in [V4_RELEASE_GATES.md](V4_RELEASE_GATES.md) are complete. +Identity-verified legal professionals can help complete that gate through the +[EvidenceBench v4 reviewer program](REVIEWING_V4.md). + v4 reports a single overall score: `0.50 × Doctrine score + 0.50 × Matter score` diff --git a/REVIEWING_V4.md b/REVIEWING_V4.md new file mode 100644 index 0000000..c0ddfc9 --- /dev/null +++ b/REVIEWING_V4.md @@ -0,0 +1,48 @@ +# Reviewing EvidenceBench v4 + +EvidenceBench v4 has 300 sealed candidate units awaiting independent +professional review. Objection Academy welcomes volunteer reviewers who are +attorneys, judges, law professors, evidence instructors, or other +identity-verifiable legal professionals with relevant experience. + +[Volunteer through the reviewer application](https://github.com/Examp-LLC/EvidenceBench/issues/new?template=evidencebench-v4-reviewer.yml) +or email `dylan@examp.com` with the subject “EvidenceBench v4 reviewer.” + +## What a reviewer does + +Each reviewer receives a private assignment containing either: + +- one applied-ruling Doctrine candidate with proposed authorities and fact + grounding; or +- one closed-universe Matter workspace with synthetic files, planted issues, + proposed findings, and atomic criteria. + +The reviewer verifies the legal result, current controlling authority, +jurisdiction, factual grounding, ambiguity, scoring tolerance, and absence of +answer leakage. A reviewer may approve, approve with recorded edits, return, +or reject a candidate. + +Most Doctrine assignments are intentionally small. Matter assignments require +reviewing six mixed-format files and take longer. + +## Confidentiality and identity + +Holdout prompts, gold annotations, matter files, and item-level responses remain +private. Do not paste an assignment into an LLM, public issue, shared document, +or external research service. Public applications must not include bar +numbers, identification documents, client information, or other sensitive +personal data. + +Objection Academy verifies identity privately and stores only an opaque +verification reference in the benchmark repository. A construction author +cannot act as the independent reviewer of the same candidate. Reviewers disclose +relevant jurisdictions and conflicts; Objection Academy retains final control +of assignment, adjudication, freeze, and release. + +## Why review matters + +Automated checks can verify hashes, schemas, coverage, and scoring invariants. +They cannot establish that a synthetic ruling is legally correct, that a state +rule is current, or that an accepted-answer set captures all sound professional +analysis. EvidenceBench will not label v4 official until every candidate has +passed its human review gate. diff --git a/models/v4/frontier-2026-07/claude-opus-5-doctrine.json b/models/v4/frontier-2026-07/claude-opus-5-doctrine.json new file mode 100644 index 0000000..a1c3221 --- /dev/null +++ b/models/v4/frontier-2026-07/claude-opus-5-doctrine.json @@ -0,0 +1,17 @@ +{ + "provider": "openrouter", + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "display_name": "Claude Opus 5", + "model_id": "anthropic/claude-opus-5", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": null, + "temperature": null, + "reasoning": {"effort": "medium", "exclude": true}, + "max_output_tokens": 1600, + "max_run_cost_usd": 7, + "timeout_seconds": 300, + "tools_enabled": false +} diff --git a/models/v4/frontier-2026-07/claude-opus-5-matter.json b/models/v4/frontier-2026-07/claude-opus-5-matter.json new file mode 100644 index 0000000..8bc3580 --- /dev/null +++ b/models/v4/frontier-2026-07/claude-opus-5-matter.json @@ -0,0 +1,18 @@ +{ + "provider": "openrouter", + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "display_name": "Claude Opus 5", + "model_id": "anthropic/claude-opus-5", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": null, + "temperature": null, + "reasoning": {"effort": "low", "exclude": true}, + "max_output_tokens": 4000, + "max_turns": 8, + "max_run_cost_usd": 37, + "timeout_seconds": 300, + "tools_enabled": true +} diff --git a/models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json b/models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json new file mode 100644 index 0000000..dec7e6c --- /dev/null +++ b/models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json @@ -0,0 +1,17 @@ +{ + "provider": "openrouter", + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "display_name": "Gemini 3.1 Pro Preview", + "model_id": "google/gemini-3.1-pro-preview", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "temperature": null, + "reasoning": {"effort": "medium", "exclude": true}, + "max_output_tokens": 1600, + "max_run_cost_usd": 3, + "timeout_seconds": 300, + "tools_enabled": false +} diff --git a/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json new file mode 100644 index 0000000..c872aa5 --- /dev/null +++ b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json @@ -0,0 +1,18 @@ +{ + "provider": "openrouter", + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "display_name": "Gemini 3.1 Pro Preview", + "model_id": "google/gemini-3.1-pro-preview", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "temperature": null, + "reasoning": {"effort": "low", "exclude": true}, + "max_output_tokens": 4000, + "max_turns": 8, + "max_run_cost_usd": 9, + "timeout_seconds": 300, + "tools_enabled": true +} diff --git a/models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json b/models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json new file mode 100644 index 0000000..8f50251 --- /dev/null +++ b/models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json @@ -0,0 +1,17 @@ +{ + "provider": "openrouter", + "display_name": "GPT-5.6 Sol", + "model_id": "openai/gpt-5.6-sol", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "temperature": null, + "reasoning": {"effort": "medium", "exclude": true}, + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "max_output_tokens": 1600, + "max_run_cost_usd": 5, + "timeout_seconds": 300, + "tools_enabled": false +} diff --git a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json new file mode 100644 index 0000000..30a7821 --- /dev/null +++ b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json @@ -0,0 +1,18 @@ +{ + "provider": "openrouter", + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "display_name": "GPT-5.6 Sol", + "model_id": "openai/gpt-5.6-sol", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "temperature": null, + "reasoning": {"effort": "low", "exclude": true}, + "max_output_tokens": 4000, + "max_turns": 8, + "max_run_cost_usd": 20, + "timeout_seconds": 300, + "tools_enabled": true +} diff --git a/models/v4/frontier-2026-07/grok-4-5-doctrine.json b/models/v4/frontier-2026-07/grok-4-5-doctrine.json new file mode 100644 index 0000000..8a2fb73 --- /dev/null +++ b/models/v4/frontier-2026-07/grok-4-5-doctrine.json @@ -0,0 +1,17 @@ +{ + "provider": "openrouter", + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "display_name": "Grok 4.5", + "model_id": "x-ai/grok-4.5", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "temperature": null, + "reasoning": {"effort": "medium", "exclude": true}, + "max_output_tokens": 1600, + "max_run_cost_usd": 3, + "timeout_seconds": 300, + "tools_enabled": false +} diff --git a/models/v4/frontier-2026-07/grok-4-5-matter.json b/models/v4/frontier-2026-07/grok-4-5-matter.json new file mode 100644 index 0000000..eb6ce85 --- /dev/null +++ b/models/v4/frontier-2026-07/grok-4-5-matter.json @@ -0,0 +1,18 @@ +{ + "provider": "openrouter", + "provider_route": {"allow_fallbacks": false, "require_parameters": true}, + "display_name": "Grok 4.5", + "model_id": "x-ai/grok-4.5", + "base_url": "https://openrouter.ai/api/v1/chat/completions", + "api_key_env": "OPENROUTER_API_KEY", + "app_title": "Objection Academy EvidenceBench v4", + "http_referer": "https://objectionacademy.com", + "seed": 20260304, + "temperature": null, + "reasoning": {"effort": "low", "exclude": true}, + "max_output_tokens": 4000, + "max_turns": 8, + "max_run_cost_usd": 9, + "timeout_seconds": 300, + "tools_enabled": true +} diff --git a/src/evidencebench/runner_v4.py b/src/evidencebench/runner_v4.py index 5cae155..2f01953 100644 --- a/src/evidencebench/runner_v4.py +++ b/src/evidencebench/runner_v4.py @@ -22,6 +22,49 @@ DEFAULT_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" +class CostBudgetExceeded(RuntimeError): + pass + + +class RunCostBudget: + def __init__(self, limit_usd: float | None) -> None: + self.limit_usd = float(limit_usd) if limit_usd is not None else None + self.spent_usd = 0.0 + + def ensure_available(self) -> None: + if self.limit_usd is not None and self.spent_usd >= self.limit_usd: + raise CostBudgetExceeded( + f"run cost limit reached: ${self.spent_usd:.6f} " + f"of ${self.limit_usd:.2f}" + ) + + def record(self, result: dict) -> None: + value = result.get("usage", {}).get("cost", 0) + if isinstance(value, (int, float, str)): + try: + self.spent_usd += float(value) + except ValueError: + pass + + +def _read_jsonl_if_present(path: Path) -> list[dict]: + if not path.is_file(): + return [] + return [ + json.loads(line) + for line in path.read_text().splitlines() + if line.strip() + ] + + +def _append_jsonl(path: Path, record: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + handle.flush() + os.fsync(handle.fileno()) + + def _load_ignored_env(start: Path) -> None: path = None for directory in (start, *start.parents): @@ -64,8 +107,14 @@ def _post_openrouter(manifest: dict, payload: dict) -> dict: headers=headers, method="POST", ) - with urllib.request.urlopen(request, timeout=manifest.get("timeout_seconds", 180)) as response: - return json.loads(response.read()) + try: + with urllib.request.urlopen( + request, timeout=manifest.get("timeout_seconds", 180) + ) as response: + return json.loads(response.read()) + except urllib.error.HTTPError as error: + error.openrouter_body = error.read().decode(errors="replace")[:2000] + raise def _is_retryable(error: Exception) -> bool: @@ -81,10 +130,22 @@ def _is_retryable(error: Exception) -> bool: } -def _request(manifest: dict, payload: dict) -> dict: +def _error_message(error: Exception) -> str: + body = getattr(error, "openrouter_body", "") + return f"{error}: {body}".strip()[:2000] + + +def _request( + manifest: dict, payload: dict, budget: RunCostBudget | None = None +) -> dict: + if budget is not None: + budget.ensure_available() for attempt in range(2): try: - return _post_openrouter(manifest, payload) + result = _post_openrouter(manifest, payload) + if budget is not None: + budget.record(result) + return result except Exception as error: if attempt == 0 and _is_retryable(error): continue @@ -92,6 +153,25 @@ def _request(manifest: dict, payload: dict) -> dict: raise AssertionError("unreachable") +def _generation_parameters(manifest: dict) -> dict: + parameters: dict = {} + if "temperature" not in manifest: + parameters["temperature"] = 0 + elif manifest["temperature"] is not None: + parameters["temperature"] = manifest["temperature"] + for key in ("reasoning", "verbosity"): + if key in manifest: + parameters[key] = manifest[key] + if "provider_route" in manifest: + parameters["provider"] = manifest["provider_route"] + seed = manifest.get("seed", 20260304) + if seed is not None: + parameters["seed"] = seed + if "parallel_tool_calls" in manifest: + parameters["parallel_tool_calls"] = manifest["parallel_tool_calls"] + return parameters + + def _json_content(message: dict) -> dict: content = message.get("content", "") if isinstance(content, list): @@ -143,54 +223,66 @@ def run_doctrine( raise ValueError("v4 runner requires provider=openrouter") if manifest.get("tools_enabled"): raise ValueError("Doctrine track must disable tools") - outputs = [] + budget = RunCostBudget(manifest.get("max_run_cost_usd")) + output = Path(output_path) + prior_outputs = _read_jsonl_if_present(output) + completed_ids = { + record.get("item_id") + for record in prior_outputs + if record.get("item_id") + } + for record in prior_outputs: + budget.record(record) + new_count = 0 for item in load_doctrine_items(items_path): + if item.id in completed_ids: + continue try: result = _request( manifest, { "model": manifest["model_id"], - "temperature": 0, - "seed": manifest.get("seed", 20260304), "max_tokens": manifest.get("max_output_tokens", 1200), "response_format": {"type": "json_object"}, "messages": [{"role": "user", "content": doctrine_prompt(item)}], + **_generation_parameters(manifest), }, + budget, ) parsed = _json_content(result["choices"][0]["message"]) - outputs.append( - { - **parsed, - "item_id": item.id, - "status": "ok", - "usage": result.get("usage", {}), - "generation_id": result.get("id"), - } - ) + record = { + **parsed, + "item_id": item.id, + "status": "ok", + "usage": result.get("usage", {}), + "generation_id": result.get("id"), + "served_model": result.get("model"), + "served_provider": result.get("provider"), + } except Exception as error: - outputs.append( - { - "item_id": item.id, - "ruling": None, - "issue_codes": [], - "authorities": [], - "grounding": [], - "confidence": None, - "explanation": "", - "status": f"failed:{type(error).__name__}", - } - ) - output = Path(output_path) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text( - "\n".join(json.dumps(record, sort_keys=True) for record in outputs) + "\n" - ) + record = { + "item_id": item.id, + "ruling": None, + "issue_codes": [], + "authorities": [], + "grounding": [], + "confidence": None, + "explanation": "", + "status": f"failed:{type(error).__name__}", + "error": _error_message(error), + } + _append_jsonl(output, record) + new_count += 1 return { "provider": "openrouter", "model": manifest["model_id"], "prompt_version": DOCTRINE_PROMPT_VERSION, "dataset_sha256": file_sha256(str(items_path)), "output_path": str(output), + "items": len(completed_ids) + new_count, + "resumed_items": len(completed_ids), + "cost_usd": budget.spent_usd, + "cost_limit_usd": budget.limit_usd, } @@ -259,6 +351,28 @@ def _extract_document(path: Path) -> str: }, }, }, + { + "type": "function", + "function": { + "name": "read_documents", + "description": ( + "Read up to ten declared matter documents in one call. Prefer " + "this when the task requires reviewing the complete record." + ), + "parameters": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "items": {"type": "string"}, + } + }, + "required": ["paths"], + }, + }, + }, { "type": "function", "function": { @@ -330,6 +444,18 @@ def execute(self, name: str, arguments: dict) -> dict: else _extract_document(path) ) return {"path": requested, "content": content} + if name == "read_documents": + paths = arguments.get("paths") + if not isinstance(paths, list) or not 1 <= len(paths) <= 10: + raise ValueError("paths must contain between one and ten documents") + if len(paths) != len(set(paths)): + raise ValueError("paths must not contain duplicates") + return { + "documents": [ + self.execute("read_document", {"path": path}) + for path in paths + ] + } if name == "search_documents": query = arguments["query"].casefold() matches = [] @@ -370,19 +496,22 @@ def execute(self, name: str, arguments: dict) -> dict: def matter_prompt(task: MatterTask) -> str: deliverables = ", ".join(task.deliverables) + documents = ", ".join(document.path for document in task.documents) return f"""You are completing the EvidenceBench v4 Matter track in a restricted workspace. Review the supplied record with the available tools. Do not browse or assume facts not in the record. Task: {task.title} Instructions: {task.instructions} +Declared documents: {documents} Required deliverables: {deliverables} You must write findings.json with this shape: {{"findings":[{{"issue_code":"...","disposition":"...", "fact_ids":["..."],"record_refs":["document:line-or-section"], "authorities":["controlling rule or case citation"],"explanation":"..."}}]}} -Use write_output for every deliverable. When complete, respond briefly. +Read every declared document. Prefer one read_documents call for the complete +record. Use write_output for every deliverable. When complete, respond briefly. """ @@ -391,6 +520,7 @@ def _run_matter_task( task_dir: Path, task: MatterTask, output_root: Path, + budget: RunCostBudget | None = None, ) -> tuple[dict, list[dict], dict]: workspace = MatterWorkspace( task_dir / "documents", @@ -405,18 +535,27 @@ def _run_matter_task( messages: list[dict] = [{"role": "user", "content": matter_prompt(task)}] transcript: list[dict] = list(messages) total_usage: dict[str, int] = {} + route_events: list[dict] = [] for _ in range(manifest.get("max_turns", 20)): result = _request( manifest, { "model": manifest["model_id"], - "temperature": 0, - "seed": manifest.get("seed", 20260304), "max_tokens": manifest.get("max_output_tokens", 2000), "messages": messages, "tools": MATTER_TOOLS, - "parallel_tool_calls": False, + **_generation_parameters(manifest), }, + budget, + ) + route_events.append( + { + "generation_id": result.get("id"), + "served_model": result.get("model"), + "served_provider": result.get("provider"), + "created": result.get("created"), + "usage": result.get("usage", {}), + } ) for key, value in result.get("usage", {}).items(): if isinstance(value, int): @@ -485,6 +624,7 @@ def _run_matter_task( "findings": parsed["findings"], "deliverables": actual_deliverables, "deliverable_metadata": deliverable_metadata, + "route_events": route_events, "status": "ok", } return response, transcript, total_usage @@ -504,8 +644,21 @@ def run_matter( raise ValueError("Matter track requires tools_enabled=true") destination = Path(output_root) destination.mkdir(parents=True, exist_ok=True) - records = [] + budget = RunCostBudget(manifest.get("max_run_cost_usd")) + response_path = destination / "responses.jsonl" + prior_records = _read_jsonl_if_present(response_path) + completed_ids = { + record.get("task_id") + for record in prior_records + if record.get("task_id") + } + for record in prior_records: + for route_event in record.get("route_events", []): + budget.record(route_event) + new_count = 0 for task_dir, task in load_matter_tasks(tasks_path): + if task.id in completed_ids: + continue task_output = destination / "workspaces" / task.id if task_output.exists(): raise FileExistsError( @@ -514,7 +667,7 @@ def run_matter( task_output.mkdir(parents=True) try: response, transcript, usage = _run_matter_task( - manifest, task_dir, task, task_output + manifest, task_dir, task, task_output, budget ) except Exception as error: response = { @@ -522,21 +675,30 @@ def run_matter( "findings": [], "deliverables": [], "status": f"failed:{type(error).__name__}", + "error": _error_message(error), } transcript = [] usage = {} - records.append(response) (destination / f"{task.id}.transcript.json").write_text( - json.dumps({"messages": transcript, "usage": usage}, indent=2) + "\n" + json.dumps( + { + "messages": transcript, + "usage": usage, + "route_events": response.get("route_events", []), + }, + indent=2, + ) + + "\n" ) - response_path = destination / "responses.jsonl" - response_path.write_text( - "\n".join(json.dumps(record, sort_keys=True) for record in records) + "\n" - ) + _append_jsonl(response_path, response) + new_count += 1 return { "provider": "openrouter", "model": manifest["model_id"], "prompt_version": MATTER_PROMPT_VERSION, "response_path": str(response_path), - "tasks": len(records), + "tasks": len(completed_ids) + new_count, + "resumed_tasks": len(completed_ids), + "cost_usd": budget.spent_usd, + "cost_limit_usd": budget.limit_usd, } diff --git a/tests/test_v4.py b/tests/test_v4.py index 8ad4571..028aa54 100644 --- a/tests/test_v4.py +++ b/tests/test_v4.py @@ -14,7 +14,13 @@ MatterResponse, MatterTask, ) -from evidencebench.runner_v4 import MatterWorkspace, doctrine_prompt +from evidencebench.runner_v4 import ( + CostBudgetExceeded, + MatterWorkspace, + RunCostBudget, + _generation_parameters, + doctrine_prompt, +) from evidencebench.release_v4 import build_release_manifest from evidencebench.scoring_v4 import score_doctrine_item, score_matter_task from evidencebench.statistics_v4 import summarize_doctrine, summarize_suite @@ -328,6 +334,12 @@ def test_workspace_exposes_canonical_text_only_through_declared_document(self) - )["content"], "canonical needle", ) + self.assertEqual( + workspace.execute( + "read_documents", {"paths": ["record.docx"]} + )["documents"][0]["content"], + "canonical needle", + ) with self.assertRaises(ValueError): workspace.execute( "read_document", {"path": "_canonical/record.txt"} @@ -347,6 +359,34 @@ def test_state_doctrine_prompt_uses_state_law(self) -> None: self.assertIn("controlling evidence law of California", prompt) self.assertNotIn("Apply the Federal Rules of Evidence", prompt) + def test_generation_parameters_support_reasoning_without_temperature(self) -> None: + self.assertEqual( + _generation_parameters({}), + {"temperature": 0, "seed": 20260304}, + ) + self.assertEqual( + _generation_parameters( + { + "temperature": None, + "reasoning": {"effort": "high"}, + "provider_route": {"allow_fallbacks": False}, + } + ), + { + "seed": 20260304, + "reasoning": {"effort": "high"}, + "provider": {"allow_fallbacks": False}, + }, + ) + + def test_cost_budget_uses_openrouter_reported_cost(self) -> None: + budget = RunCostBudget(1.0) + budget.record({"usage": {"cost": 0.4}}) + budget.ensure_available() + budget.record({"usage": {"cost": "0.6"}}) + with self.assertRaises(CostBudgetExceeded): + budget.ensure_available() + if __name__ == "__main__": unittest.main() From 748c9de28da12391e5fab385f84b94161db7f650 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 13:43:26 -0700 Subject: [PATCH 4/8] Define v4 research release protocol --- BENCHMARK_CARD_V4.md | 10 +- HARVEY_LAB_COMPARISON.md | 17 +-- METHODOLOGY_V4.md | 20 +++- README.md | 15 +-- V4_RELEASE_GATES.md | 5 + .../claude-opus-5-doctrine.json | 1 + .../claude-opus-5-matter.json | 1 + .../gemini-3-1-pro-doctrine.json | 1 + .../gemini-3-1-pro-matter.json | 1 + .../gpt-5-6-sol-doctrine.json | 1 + .../frontier-2026-07/gpt-5-6-sol-matter.json | 1 + .../frontier-2026-07/grok-4-5-doctrine.json | 1 + .../v4/frontier-2026-07/grok-4-5-matter.json | 1 + results/v4-research-2026-07/README.md | 22 ++++ scripts/publish_v4_research_snapshot.py | 106 ++++++++++++++++++ src/evidencebench/runner_v4.py | 76 +++++++++---- tests/test_v4.py | 52 +++++++++ 17 files changed, 285 insertions(+), 46 deletions(-) create mode 100644 results/v4-research-2026-07/README.md create mode 100644 scripts/publish_v4_research_snapshot.py diff --git a/BENCHMARK_CARD_V4.md b/BENCHMARK_CARD_V4.md index 2d4db06..15d72b7 100644 --- a/BENCHMARK_CARD_V4.md +++ b/BENCHMARK_CARD_V4.md @@ -26,10 +26,11 @@ leaderboard. ## Private data -The official set will remain under Objection Academy control in a private +The corpus remains under Objection Academy control in a private source-controlled repository. A public release commits to it by cryptographic hash and reports counts and coverage without releasing answers or item-level -outputs. +outputs. The July 2026 research corpus contains 240 Doctrine candidates and 60 +Matter candidates; none had completed professional review at run time. ## Primary metric @@ -54,6 +55,11 @@ strict secondary metric. - uncertainty is clustered by task family; and - official results require repeated runs and disclosure of failures and costs. +The July 2026 snapshot does not satisfy the independent-review or repeated-run +controls. It is published only as an unreviewed research release so researchers +can inspect the evaluator and initial measurement behavior while professional +review remains open. + ## Foreseeable risks Contamination can inflate results. Synthetic matters may not capture the diff --git a/HARVEY_LAB_COMPARISON.md b/HARVEY_LAB_COMPARISON.md index 9e34ddb..960dd68 100644 --- a/HARVEY_LAB_COMPARISON.md +++ b/HARVEY_LAB_COMPARISON.md @@ -7,12 +7,12 @@ counts are descriptive rather than permanent. |---|---|---| | Primary target | Long-horizon legal work across many practice areas | Evidence-law doctrine and litigation-record analysis | | Unit | Matter files, instruction, and reviewable work product | Doctrine item or closed-universe evidence matter | -| Breadth | Current repository documentation reports 1,660 tasks and roughly 101,000 criteria | v4 candidate has 8 public Doctrine and 2 public Matter development fixtures; official set is not yet authored | +| Breadth | Harvey's May 2026 announcement reported more than 1,200 tasks, 24 practice areas, and 75,000 expert-written criteria | Private research corpus has 240 Doctrine items and 60 Matter tasks; 8 Doctrine and 2 Matter fixtures are public | | Tools | File read/edit/write, glob, grep, bash, and document-format skills | List/read/search documents and write outputs; no model shell or network | | Core grading | Semantic LLM judge for each binary criterion; task score uses all-pass | Deterministic structured criteria; weighted track score plus all-critical Matter resolution | | Citation treatment | A rubric can require citations | Citations are normalized against a pinned authority corpus; invalid, nonexistent, and unsupported authorities are separated | | Uncertainty | Results emphasize all-pass and criterion pass rates | Family-clustered 95% bootstrap intervals, repeated-run requirement, and weight sensitivity | -| Contamination strategy | Public tasks plus a held-out set | Public DRAFT development set plus a private family-separated holdout with public cryptographic commitments | +| Contamination strategy | Public tasks plus a held-out test set | Public DRAFT development set plus a private family-separated candidate corpus with a public cryptographic commitment | ## Lessons adopted @@ -66,13 +66,14 @@ v4 addresses those problems with: - one preregistered overall score with mandatory track disclosure; and - clustered confidence intervals, run replication, and weighting sensitivity. -## What remains before a defensible launch +## What remains before an official launch -The v4 software is not the v4 benchmark corpus. The official sample needs a -coverage matrix, original families, independent professional review, -adjudication, pilot-based item analysis, and a preregistered release plan. -Only after those gates pass should Objection Academy run paid models or publish -a leaderboard. +The candidate corpus has a frozen coverage matrix, original families, private +provenance, review packets, and deterministic validation. It still needs +independent professional review, adjudication, replicated model runs, external +reproduction, and prospective challenge handling. Until those gates pass, the +July 2026 results remain an unreviewed research release rather than an official +leaderboard. ## Sources diff --git a/METHODOLOGY_V4.md b/METHODOLOGY_V4.md index 3a3a12b..476ee3b 100644 --- a/METHODOLOGY_V4.md +++ b/METHODOLOGY_V4.md @@ -1,7 +1,8 @@ # EvidenceBench v4 methodology -Status: implementation candidate. The public development fixtures are `DRAFT`, -not an official benchmark release. +Status: evaluator-complete. The July 2026 four-model snapshot is an +**unreviewed research release**, not an official or attorney-validated +leaderboard. The official gates remain unchanged. ## Research question @@ -94,8 +95,10 @@ the v4 score. Model sampling is temperature zero with a recorded seed when the route supports it. Each official model result must include at least three independent runs. -The leaderboard reports the run mean, run dispersion, and family-clustered 95% -bootstrap confidence intervals. +The July 2026 research release uses one run per model and reports +family-clustered 95% bootstrap confidence intervals across the frozen corpus; +those intervals measure item-family sampling uncertainty, not run-to-run model +variance. ## Dataset design @@ -110,7 +113,9 @@ records have hashed canonical-text companions so extraction-library variance cannot change the input presented to a model. Real cases may be used only when the source, license, holding, current-law status, and retrieval hash are recorded. Every official item must name an author and at least one -independent reviewer and must have status `APPROVED`. +independent reviewer and must have status `APPROVED`. The July 2026 research +corpus is frozen while all 300 items remain `DRAFT`; its public results must +carry that limitation wherever they appear. The construction target is 240 Doctrine items and 60 Matter tasks. Doctrine contains 151 migrated v3 concepts and 89 new coverage items, scored as 168 @@ -136,6 +141,11 @@ An official result publishes: - family-clustered confidence intervals and weighting sensitivity; and - all deviations from the reference harness. +The July 2026 research release publishes the same aggregate fields where +available, plus an explicit zero-of-300 review count, one-run disclosure, and +`unreviewed_research_release` status. Item-level prompts, annotations, +responses, transcripts, and scores remain private. + ## Known limitations EvidenceBench focuses on United States federal evidence doctrine and a diff --git a/README.md b/README.md index 8eebeb6..20703af 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,12 @@ project, not legal advice or a substitute for professional legal research. ## v4 status -The v4 evaluator, schemas, OpenRouter runners, scoring, integrity checks, and -public development fixtures are implemented. The fixtures are deliberately -marked `DRAFT`; they validate for development and fail the `--official` gate. -EvidenceBench v4 must not be presented as an official leaderboard until the +The v4 evaluator, schemas, OpenRouter runners, scoring, integrity checks, public +development fixtures, and private 300-item candidate corpus are implemented. +The July 2026 four-model snapshot is an **unreviewed research release**: all +gold labels remain `DRAFT`, zero professional reviews were complete at run +time, and only one run was made per model. EvidenceBench v4 must not be +presented as an official or attorney-validated leaderboard until the legal-review and release gates in [V4_RELEASE_GATES.md](V4_RELEASE_GATES.md) are complete. @@ -38,8 +40,7 @@ specific lessons taken from Harvey LAB are in - A 24-question development set and its scoring annotations. - The prompt contract, scoring code, model manifests, methodology, and website export format. -- Aggregate official results after an approved sealed holdout run, including - separate FRE and caselaw dimensions. +- Aggregate research or official results with their release and review status. ## What is sealed @@ -57,7 +58,7 @@ annotations, and item-level results are not public. Once approved, published manifests will commit to the holdout with a SHA-256 hash, counts, categories, protocol hash, and aggregate metrics. -The private v4 construction target is 240 Doctrine candidates and 60 Matter +The private v4 research corpus contains 240 Doctrine candidates and 60 Matter tasks. These remain a candidate pool—not an official holdout—until every item passes the review and freeze gates. diff --git a/V4_RELEASE_GATES.md b/V4_RELEASE_GATES.md index 25ff74d..60e30e3 100644 --- a/V4_RELEASE_GATES.md +++ b/V4_RELEASE_GATES.md @@ -2,6 +2,11 @@ No result is “official EvidenceBench v4” until all gates pass. +The July 2026 four-model snapshot is permitted only under the distinct label +`unreviewed_research_release`. It does not waive, satisfy, or weaken any gate +below and must not be described as an official or attorney-validated +leaderboard. + ## Data gate - [ ] Target sample and coverage plan are frozen before model runs. diff --git a/models/v4/frontier-2026-07/claude-opus-5-doctrine.json b/models/v4/frontier-2026-07/claude-opus-5-doctrine.json index a1c3221..dbc2897 100644 --- a/models/v4/frontier-2026-07/claude-opus-5-doctrine.json +++ b/models/v4/frontier-2026-07/claude-opus-5-doctrine.json @@ -11,6 +11,7 @@ "temperature": null, "reasoning": {"effort": "medium", "exclude": true}, "max_output_tokens": 1600, + "concurrency": 3, "max_run_cost_usd": 7, "timeout_seconds": 300, "tools_enabled": false diff --git a/models/v4/frontier-2026-07/claude-opus-5-matter.json b/models/v4/frontier-2026-07/claude-opus-5-matter.json index 8bc3580..1b37fd0 100644 --- a/models/v4/frontier-2026-07/claude-opus-5-matter.json +++ b/models/v4/frontier-2026-07/claude-opus-5-matter.json @@ -12,6 +12,7 @@ "reasoning": {"effort": "low", "exclude": true}, "max_output_tokens": 4000, "max_turns": 8, + "concurrency": 2, "max_run_cost_usd": 37, "timeout_seconds": 300, "tools_enabled": true diff --git a/models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json b/models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json index dec7e6c..1f50ed7 100644 --- a/models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json +++ b/models/v4/frontier-2026-07/gemini-3-1-pro-doctrine.json @@ -11,6 +11,7 @@ "temperature": null, "reasoning": {"effort": "medium", "exclude": true}, "max_output_tokens": 1600, + "concurrency": 3, "max_run_cost_usd": 3, "timeout_seconds": 300, "tools_enabled": false diff --git a/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json index c872aa5..eb17a6b 100644 --- a/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json +++ b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json @@ -12,6 +12,7 @@ "reasoning": {"effort": "low", "exclude": true}, "max_output_tokens": 4000, "max_turns": 8, + "concurrency": 2, "max_run_cost_usd": 9, "timeout_seconds": 300, "tools_enabled": true diff --git a/models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json b/models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json index 8f50251..3211ca5 100644 --- a/models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json +++ b/models/v4/frontier-2026-07/gpt-5-6-sol-doctrine.json @@ -11,6 +11,7 @@ "reasoning": {"effort": "medium", "exclude": true}, "provider_route": {"allow_fallbacks": false, "require_parameters": true}, "max_output_tokens": 1600, + "concurrency": 3, "max_run_cost_usd": 5, "timeout_seconds": 300, "tools_enabled": false diff --git a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json index 30a7821..bc4f13b 100644 --- a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json +++ b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json @@ -12,6 +12,7 @@ "reasoning": {"effort": "low", "exclude": true}, "max_output_tokens": 4000, "max_turns": 8, + "concurrency": 2, "max_run_cost_usd": 20, "timeout_seconds": 300, "tools_enabled": true diff --git a/models/v4/frontier-2026-07/grok-4-5-doctrine.json b/models/v4/frontier-2026-07/grok-4-5-doctrine.json index 8a2fb73..3c29ba3 100644 --- a/models/v4/frontier-2026-07/grok-4-5-doctrine.json +++ b/models/v4/frontier-2026-07/grok-4-5-doctrine.json @@ -11,6 +11,7 @@ "temperature": null, "reasoning": {"effort": "medium", "exclude": true}, "max_output_tokens": 1600, + "concurrency": 3, "max_run_cost_usd": 3, "timeout_seconds": 300, "tools_enabled": false diff --git a/models/v4/frontier-2026-07/grok-4-5-matter.json b/models/v4/frontier-2026-07/grok-4-5-matter.json index eb6ce85..4c6e65f 100644 --- a/models/v4/frontier-2026-07/grok-4-5-matter.json +++ b/models/v4/frontier-2026-07/grok-4-5-matter.json @@ -12,6 +12,7 @@ "reasoning": {"effort": "low", "exclude": true}, "max_output_tokens": 4000, "max_turns": 8, + "concurrency": 2, "max_run_cost_usd": 9, "timeout_seconds": 300, "tools_enabled": true diff --git a/results/v4-research-2026-07/README.md b/results/v4-research-2026-07/README.md new file mode 100644 index 0000000..74f67cd --- /dev/null +++ b/results/v4-research-2026-07/README.md @@ -0,0 +1,22 @@ +# EvidenceBench v4 July 2026 research snapshot + +This directory contains aggregate-only results for four frontier model families +evaluated on the frozen 240-item Doctrine and 60-task Matter candidate corpus. + +This is an **unreviewed research release**, not an official or +attorney-validated leaderboard: + +- zero of 300 gold annotations had completed independent professional review; +- each model was run once, so no run-to-run variance estimate is available; +- item-level prompts, gold annotations, responses, transcripts, deliverables, + and scores remain sealed; +- confidence intervals reflect family-clustered corpus resampling only; and +- all calls used OpenRouter with provider fallback disabled. + +`aggregate-results.json` will contain the corpus commitment, exact model IDs, +parameters, cost and failure disclosures, track metrics, authority-error counts, +confidence intervals, and the single 50/50 overall score. + +The evaluator, model manifests, methodology, and public development fixtures +are in this repository. Professional reviewers may volunteer through +[the v4 reviewer program](../../REVIEWING_V4.md). diff --git a/scripts/publish_v4_research_snapshot.py b/scripts/publish_v4_research_snapshot.py new file mode 100644 index 0000000..88e1d51 --- /dev/null +++ b/scripts/publish_v4_research_snapshot.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Create public aggregate and website payloads from the private v4 run.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +REPOSITORY_URL = "https://github.com/Examp-LLC/EvidenceBench" +METHODOLOGY_URL = f"{REPOSITORY_URL}/blob/main/METHODOLOGY_V4.md" +REVIEWER_URL = ( + f"{REPOSITORY_URL}/issues/new" + "?template=evidencebench-v4-reviewer.yml" +) + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def public_payload(private: dict) -> dict: + if private.get("release_status") != "unreviewed_research_release": + raise ValueError("input is not the expected unreviewed research release") + corpus = private["corpus"] + cost = private["cost_accounting"] + return { + "schema_version": "4.0", + "release_id": private["release_id"], + "release_status": private["release_status"], + "published_as_official": False, + "review_status": private["review_status"], + "corpus": { + "doctrine": corpus["doctrine"], + "matter": corpus["matter"], + "combined_candidate_tree_sha256": corpus[ + "combined_candidate_tree_sha256" + ], + }, + "protocol": private["protocol"], + "cost_accounting": { + "hard_limit_usd": cost["hard_limit_usd"], + "account_delta_usd": cost["account_delta_usd"], + "full_run_reported_cost_usd": cost["full_run_reported_cost_usd"], + "smoke_reported_cost_usd": cost["smoke_reported_cost_usd"], + }, + "models": private["models"], + } + + +def website_payload(public: dict) -> dict: + return { + "release_status": public["release_status"], + "benchmark_version": "4.0.0-research", + "published_at": f"{public['protocol']['run_date']}T00:00:00Z", + "repository_url": REPOSITORY_URL, + "methodology_url": METHODOLOGY_URL, + "prior_version_url": "/articles/evidencebench-public-good-benchmark", + "article_url": "/articles/evidencebench-v4-frontier-models", + "reviewer_url": REVIEWER_URL, + "review_status": public["review_status"], + "corpus": { + "doctrine_items": public["corpus"]["doctrine"]["count"], + "doctrine_families": public["corpus"]["doctrine"]["families"], + "doctrine_domains": 12, + "matter_tasks": public["corpus"]["matter"]["count"], + "documents_per_task": 6, + "total_items": ( + public["corpus"]["doctrine"]["count"] + + public["corpus"]["matter"]["count"] + ), + "commitment_sha256": public["corpus"][ + "combined_candidate_tree_sha256" + ], + }, + "protocol": { + "provider": public["protocol"]["provider"], + "runs_per_model": public["protocol"]["runs_per_model"], + "overall_formula": "0.50 × Doctrine + 0.50 × Matter", + "doctrine_tools_enabled": False, + "matter_tools_enabled": True, + }, + "models": public["models"], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--private-aggregate", type=Path, required=True) + parser.add_argument("--public-output", type=Path, required=True) + parser.add_argument("--website-output", type=Path, required=True) + args = parser.parse_args() + private = json.loads(args.private_aggregate.read_text()) + public = public_payload(private) + website = website_payload(public) + write_json(args.public_output, public) + write_json(args.website_output, website) + print(f"published aggregate: {args.public_output}") + print(f"published website payload: {args.website_output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/evidencebench/runner_v4.py b/src/evidencebench/runner_v4.py index 2f01953..08b5dc2 100644 --- a/src/evidencebench/runner_v4.py +++ b/src/evidencebench/runner_v4.py @@ -6,9 +6,11 @@ import re import socket import subprocess +import threading import urllib.error import urllib.request import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from xml.etree import ElementTree @@ -30,23 +32,29 @@ class RunCostBudget: def __init__(self, limit_usd: float | None) -> None: self.limit_usd = float(limit_usd) if limit_usd is not None else None self.spent_usd = 0.0 + self._lock = threading.Lock() def ensure_available(self) -> None: - if self.limit_usd is not None and self.spent_usd >= self.limit_usd: - raise CostBudgetExceeded( - f"run cost limit reached: ${self.spent_usd:.6f} " - f"of ${self.limit_usd:.2f}" - ) + with self._lock: + if self.limit_usd is not None and self.spent_usd >= self.limit_usd: + raise CostBudgetExceeded( + f"run cost limit reached: ${self.spent_usd:.6f} " + f"of ${self.limit_usd:.2f}" + ) def record(self, result: dict) -> None: value = result.get("usage", {}).get("cost", 0) if isinstance(value, (int, float, str)): try: - self.spent_usd += float(value) + with self._lock: + self.spent_usd += float(value) except ValueError: pass +_JSONL_LOCK = threading.Lock() + + def _read_jsonl_if_present(path: Path) -> list[dict]: if not path.is_file(): return [] @@ -59,10 +67,11 @@ def _read_jsonl_if_present(path: Path) -> list[dict]: def _append_jsonl(path: Path, record: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a") as handle: - handle.write(json.dumps(record, sort_keys=True) + "\n") - handle.flush() - os.fsync(handle.fileno()) + with _JSONL_LOCK: + with path.open("a") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + handle.flush() + os.fsync(handle.fileno()) def _load_ignored_env(start: Path) -> None: @@ -233,10 +242,13 @@ def run_doctrine( } for record in prior_outputs: budget.record(record) - new_count = 0 - for item in load_doctrine_items(items_path): - if item.id in completed_ids: - continue + pending_items = [ + item + for item in load_doctrine_items(items_path) + if item.id not in completed_ids + ] + + def evaluate(item) -> dict: try: result = _request( manifest, @@ -250,7 +262,7 @@ def run_doctrine( budget, ) parsed = _json_content(result["choices"][0]["message"]) - record = { + return { **parsed, "item_id": item.id, "status": "ok", @@ -260,7 +272,7 @@ def run_doctrine( "served_provider": result.get("provider"), } except Exception as error: - record = { + return { "item_id": item.id, "ruling": None, "issue_codes": [], @@ -271,8 +283,13 @@ def run_doctrine( "status": f"failed:{type(error).__name__}", "error": _error_message(error), } - _append_jsonl(output, record) - new_count += 1 + workers = max(1, int(manifest.get("concurrency", 1))) + new_count = 0 + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(evaluate, item) for item in pending_items] + for future in as_completed(futures): + _append_jsonl(output, future.result()) + new_count += 1 return { "provider": "openrouter", "model": manifest["model_id"], @@ -655,10 +672,14 @@ def run_matter( for record in prior_records: for route_event in record.get("route_events", []): budget.record(route_event) - new_count = 0 - for task_dir, task in load_matter_tasks(tasks_path): - if task.id in completed_ids: - continue + pending_tasks = [ + (task_dir, task) + for task_dir, task in load_matter_tasks(tasks_path) + if task.id not in completed_ids + ] + + def evaluate(entry) -> dict: + task_dir, task = entry task_output = destination / "workspaces" / task.id if task_output.exists(): raise FileExistsError( @@ -690,8 +711,15 @@ def run_matter( ) + "\n" ) - _append_jsonl(response_path, response) - new_count += 1 + return response + + workers = max(1, int(manifest.get("concurrency", 1))) + new_count = 0 + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(evaluate, entry) for entry in pending_tasks] + for future in as_completed(futures): + _append_jsonl(response_path, future.result()) + new_count += 1 return { "provider": "openrouter", "model": manifest["model_id"], diff --git a/tests/test_v4.py b/tests/test_v4.py index 028aa54..e0bb2fe 100644 --- a/tests/test_v4.py +++ b/tests/test_v4.py @@ -1,8 +1,10 @@ from __future__ import annotations +import json import tempfile import unittest from pathlib import Path +from unittest.mock import patch from evidencebench.datasets_v4 import load_doctrine_items, load_matter_tasks from evidencebench.models_v4 import ( @@ -20,6 +22,7 @@ RunCostBudget, _generation_parameters, doctrine_prompt, + run_doctrine, ) from evidencebench.release_v4 import build_release_manifest from evidencebench.scoring_v4 import score_doctrine_item, score_matter_task @@ -387,6 +390,55 @@ def test_cost_budget_uses_openrouter_reported_cost(self) -> None: with self.assertRaises(CostBudgetExceeded): budget.ensure_available() + def test_doctrine_run_resumes_without_repeating_paid_calls(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifest = root / "manifest.json" + items = root / "items.jsonl" + output = root / "responses.jsonl" + manifest.write_text( + json.dumps( + { + "provider": "openrouter", + "model_id": "test/model", + "tools_enabled": False, + "max_run_cost_usd": 1, + } + ) + ) + items.write_text(DOCTRINE.read_text().splitlines()[0] + "\n") + response = { + "id": "generation-1", + "model": "test/model", + "provider": "Test", + "choices": [ + { + "message": { + "content": json.dumps( + { + "ruling": "exclude", + "issue_codes": [], + "authorities": [], + "grounding": [], + "confidence": 0.5, + "explanation": "test", + } + ) + } + } + ], + "usage": {"cost": 0.01}, + } + with patch( + "evidencebench.runner_v4._request", return_value=response + ) as request: + first = run_doctrine(manifest, items, output) + second = run_doctrine(manifest, items, output) + self.assertEqual(request.call_count, 1) + self.assertEqual(first["items"], 1) + self.assertEqual(second["resumed_items"], 1) + self.assertEqual(len(output.read_text().splitlines()), 1) + if __name__ == "__main__": unittest.main() From a2860299f43f6eeb45e4e32126f6f5ce8c261052 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 13:53:49 -0700 Subject: [PATCH 5/8] Fix v4 structured scoring contract --- METHODOLOGY_V4.md | 7 +++ .../claude-opus-5-doctrine.json | 2 +- .../claude-opus-5-matter.json | 2 +- .../gemini-3-1-pro-matter.json | 2 +- .../frontier-2026-07/gpt-5-6-sol-matter.json | 2 +- .../v4/frontier-2026-07/grok-4-5-matter.json | 2 +- src/evidencebench/runner_v4.py | 53 +++++++++++++++++-- src/evidencebench/scoring_v4.py | 28 ++++++++-- tests/test_v4.py | 36 +++++++++++++ 9 files changed, 121 insertions(+), 13 deletions(-) diff --git a/METHODOLOGY_V4.md b/METHODOLOGY_V4.md index 476ee3b..8fd7fcd 100644 --- a/METHODOLOGY_V4.md +++ b/METHODOLOGY_V4.md @@ -36,6 +36,13 @@ The fixed equal weighting prevents the larger track from silently dominating. Every release also reports 40/60 and 60/40 sensitivity values. A submission must complete both tracks; a missing track does not receive a headline score. +Doctrine prompts publish a fixed twelve-code issue catalog. Models select every +materially controlling domain code and use the same code to ground supplied fact +IDs. Matter prompts publish neutral issue-slot identifiers for every required +finding. Those identifiers disclose output cardinality, not the disposition, +authority, fact, or record answer. Publishing both contracts prevents opaque +private IDs from making structured-match credit impossible. + ## Doctrine scoring Each item receives: diff --git a/models/v4/frontier-2026-07/claude-opus-5-doctrine.json b/models/v4/frontier-2026-07/claude-opus-5-doctrine.json index dbc2897..b82450c 100644 --- a/models/v4/frontier-2026-07/claude-opus-5-doctrine.json +++ b/models/v4/frontier-2026-07/claude-opus-5-doctrine.json @@ -12,7 +12,7 @@ "reasoning": {"effort": "medium", "exclude": true}, "max_output_tokens": 1600, "concurrency": 3, - "max_run_cost_usd": 7, + "max_run_cost_usd": 5, "timeout_seconds": 300, "tools_enabled": false } diff --git a/models/v4/frontier-2026-07/claude-opus-5-matter.json b/models/v4/frontier-2026-07/claude-opus-5-matter.json index 1b37fd0..f6fc742 100644 --- a/models/v4/frontier-2026-07/claude-opus-5-matter.json +++ b/models/v4/frontier-2026-07/claude-opus-5-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 37, + "max_run_cost_usd": 34, "timeout_seconds": 300, "tools_enabled": true } diff --git a/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json index eb17a6b..5c6783c 100644 --- a/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json +++ b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 9, + "max_run_cost_usd": 8, "timeout_seconds": 300, "tools_enabled": true } diff --git a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json index bc4f13b..97da8f1 100644 --- a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json +++ b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 20, + "max_run_cost_usd": 18, "timeout_seconds": 300, "tools_enabled": true } diff --git a/models/v4/frontier-2026-07/grok-4-5-matter.json b/models/v4/frontier-2026-07/grok-4-5-matter.json index 4c6e65f..120cafa 100644 --- a/models/v4/frontier-2026-07/grok-4-5-matter.json +++ b/models/v4/frontier-2026-07/grok-4-5-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 9, + "max_run_cost_usd": 8, "timeout_seconds": 300, "tools_enabled": true } diff --git a/src/evidencebench/runner_v4.py b/src/evidencebench/runner_v4.py index 08b5dc2..1701105 100644 --- a/src/evidencebench/runner_v4.py +++ b/src/evidencebench/runner_v4.py @@ -19,9 +19,41 @@ from .models_v4 import MatterTask -DOCTRINE_PROMPT_VERSION = "evidencebench-v4-doctrine-1" -MATTER_PROMPT_VERSION = "evidencebench-v4-matter-agent-1" +DOCTRINE_PROMPT_VERSION = "evidencebench-v4-doctrine-2" +MATTER_PROMPT_VERSION = "evidencebench-v4-matter-agent-2" DEFAULT_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" +DOCTRINE_ISSUE_CATALOG = ( + ( + "EBV4_D01_JUDICIAL_ADMINISTRATION", + "Judicial administration and preliminary questions", + ), + ("EBV4_D02_RELEVANCE_403", "Relevance and Rule 403 balancing"), + ( + "EBV4_D03_CHARACTER_PROPENSITY_HABIT", + "Character, propensity, other acts, or habit", + ), + ("EBV4_D04_POLICY_EXCLUSIONS", "Policy-based exclusions and compromise rules"), + ( + "EBV4_D05_WITNESS_EXAMINATION", + "Witness competency, examination, and sequestration", + ), + ("EBV4_D06_IMPEACHMENT_REHABILITATION", "Impeachment and rehabilitation"), + ("EBV4_D07_OPINION_EXPERTS", "Lay or expert opinion evidence"), + ("EBV4_D08_HEARSAY_DEFINITIONS", "Hearsay definitions and exclusions"), + ("EBV4_D09_HEARSAY_EXCEPTIONS", "Hearsay exceptions"), + ( + "EBV4_D10_AUTHENTICATION_IDENTIFICATION", + "Authentication and identification", + ), + ( + "EBV4_D11_CONTENTS_ORIGINALS_SUMMARIES", + "Original-writing rule, contents, and summaries", + ), + ( + "EBV4_D12_PRIVILEGE_CONSTITUTIONAL", + "Privilege and constitutional evidence limits", + ), +) class CostBudgetExceeded(RuntimeError): @@ -199,6 +231,9 @@ def _json_content(message: dict) -> dict: def doctrine_prompt(item) -> str: facts = "\n".join(f"{fact.id}: {fact.text}" for fact in item.facts) rulings = ", ".join(item.allowed_rulings) + issue_catalog = "\n".join( + f"- {code}: {label}" for code, label in DOCTRINE_ISSUE_CATALOG + ) governing_law = ( "the Federal Rules of Evidence" if item.jurisdiction == "federal" @@ -209,11 +244,13 @@ def doctrine_prompt(item) -> str: return one JSON object only. Use only the supplied fact IDs. Required shape: -{{"ruling":"admit|exclude|limit|defer","issue_codes":["RULE_..."], -"authorities":["controlling rule or case citation"],"grounding":[{{"issue_code":"RULE_...", +{{"ruling":"admit|exclude|limit|defer","issue_codes":["EBV4_D..."], +"authorities":["controlling rule or case citation"],"grounding":[{{"issue_code":"EBV4_D...", "fact_ids":["F1"]}}],"confidence":0.0,"explanation":"brief analysis"}} Allowed rulings: {rulings} +Choose every materially controlling issue code from this fixed catalog: +{issue_catalog} Question: {item.stem} Facts: {facts} @@ -514,6 +551,9 @@ def execute(self, name: str, arguments: dict) -> dict: def matter_prompt(task: MatterTask) -> str: deliverables = ", ".join(task.deliverables) documents = ", ".join(document.path for document in task.documents) + issue_codes = ", ".join( + finding.issue_code for finding in task.gold_findings + ) return f"""You are completing the EvidenceBench v4 Matter track in a restricted workspace. Review the supplied record with the available tools. Do not browse or assume facts not in the record. @@ -522,11 +562,16 @@ def matter_prompt(task: MatterTask) -> str: Instructions: {task.instructions} Declared documents: {documents} Required deliverables: {deliverables} +Required issue slots: {issue_codes} You must write findings.json with this shape: {{"findings":[{{"issue_code":"...","disposition":"...", "fact_ids":["..."],"record_refs":["document:line-or-section"], "authorities":["controlling rule or case citation"],"explanation":"..."}}]}} +Write exactly one finding for every required issue slot. The slot identifiers +are neutral output keys, not answers; determine each disposition, authority, +fact, and record reference from the matter. +Disposition must be exactly one of: admit, exclude, limit, defer. Read every declared document. Prefer one read_documents call for the complete record. Use write_output for every deliverable. When complete, respond briefly. """ diff --git a/src/evidencebench/scoring_v4.py b/src/evidencebench/scoring_v4.py index ec747f7..7248f2b 100644 --- a/src/evidencebench/scoring_v4.py +++ b/src/evidencebench/scoring_v4.py @@ -52,6 +52,10 @@ def _precision_recall_f1( return precision, recall, f1 +def _authority_matches(submitted: str, accepted: str) -> bool: + return submitted == accepted or submitted.startswith(f"{accepted}(") + + def _authority_score( submitted: Iterable[str], accepted_values: Iterable[str], @@ -76,7 +80,9 @@ def _authority_score( if (normalized := normalize_citation(value)) is not None } supported = { - value for value in canonical_submissions if value in accepted + value + for value in canonical_submissions + if any(_authority_matches(value, target) for target in accepted) } hallucinated = sorted( original @@ -106,7 +112,14 @@ def _authority_score( for group in required_groups ] recall = ( - sum(bool(group.intersection(supported)) for group in normalized_groups) + sum( + any( + _authority_matches(submitted, target) + for submitted in supported + for target in group + ) + for group in normalized_groups + ) / len(normalized_groups) if normalized_groups else 1.0 @@ -358,7 +371,10 @@ def _matter_authority_f1( submitted_count += 1 if normalized is None: invalid.append(raw) - elif normalized in accepted: + elif any( + _authority_matches(normalized, target) + for target in accepted + ): supported_count += 1 supported_by_issue[issue].add(normalized) elif not citation_exists(normalized): @@ -374,7 +390,11 @@ def _matter_authority_f1( if (normalized := normalize_citation(value)) is not None } required_total += 1 - required_met += bool(normalized_group & supported_by_issue[issue]) + required_met += any( + _authority_matches(submitted, target) + for submitted in supported_by_issue[issue] + for target in normalized_group + ) recall = required_met / required_total if required_total else 1.0 f1 = ( 0.0 diff --git a/tests/test_v4.py b/tests/test_v4.py index e0bb2fe..9ad64cd 100644 --- a/tests/test_v4.py +++ b/tests/test_v4.py @@ -22,6 +22,7 @@ RunCostBudget, _generation_parameters, doctrine_prompt, + matter_prompt, run_doctrine, ) from evidencebench.release_v4 import build_release_manifest @@ -97,6 +98,24 @@ def test_invalid_authority_is_not_silently_dropped(self) -> None: self.assertEqual(score.authority_precision, 0.5) self.assertLess(score.doctrine_score, 1.0) + def test_specific_subsection_satisfies_accepted_parent_rule(self) -> None: + source = load_doctrine_items(DOCTRINE)[0] + payload = source.to_dict() + payload["gold"]["accepted_authorities"] = ["FRE 104"] + payload["gold"]["required_authority_groups"] = [["FRE 104"]] + item = source.__class__.from_dict(payload) + response = DoctrineResponse( + item_id=item.id, + ruling=item.gold.ruling, + issue_codes=item.gold.issue_codes, + authorities=["FRE 104(a)"], + grounding=item.gold.grounding, + confidence=1.0, + ) + score = score_doctrine_item(item, response) + self.assertEqual(score.authority_precision, 1.0) + self.assertEqual(score.authority_recall, 1.0) + def test_perfect_matter_response_scores_one(self) -> None: _, task = load_matter_tasks(MATTER)[0] response = MatterResponse( @@ -362,6 +381,23 @@ def test_state_doctrine_prompt_uses_state_law(self) -> None: self.assertIn("controlling evidence law of California", prompt) self.assertNotIn("Apply the Federal Rules of Evidence", prompt) + def test_doctrine_prompt_exposes_the_fixed_issue_code_catalog(self) -> None: + prompt = doctrine_prompt(load_doctrine_items(DOCTRINE)[0]) + self.assertIn("EBV4_D01_JUDICIAL_ADMINISTRATION", prompt) + self.assertIn("EBV4_D12_PRIVILEGE_CONSTITUTIONAL", prompt) + self.assertIn("Choose every materially controlling issue code", prompt) + + def test_matter_prompt_exposes_neutral_required_issue_slots(self) -> None: + _, task = load_matter_tasks(MATTER)[0] + prompt = matter_prompt(task) + for finding in task.gold_findings: + self.assertIn(finding.issue_code, prompt) + self.assertIn("neutral output keys, not answers", prompt) + self.assertIn( + "Disposition must be exactly one of: admit, exclude, limit, defer", + prompt, + ) + def test_generation_parameters_support_reasoning_without_temperature(self) -> None: self.assertEqual( _generation_parameters({}), From 8012350d87d918711d5a3416bd5476a951cdeccf Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 14:06:18 -0700 Subject: [PATCH 6/8] Make v4 matter runs safely resumable --- models/v4/frontier-2026-07/claude-opus-5-matter.json | 2 +- models/v4/frontier-2026-07/gemini-3-1-pro-matter.json | 2 +- models/v4/frontier-2026-07/gpt-5-6-sol-matter.json | 2 +- src/evidencebench/runner_v4.py | 11 ++++++++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/models/v4/frontier-2026-07/claude-opus-5-matter.json b/models/v4/frontier-2026-07/claude-opus-5-matter.json index f6fc742..cc08fe2 100644 --- a/models/v4/frontier-2026-07/claude-opus-5-matter.json +++ b/models/v4/frontier-2026-07/claude-opus-5-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 34, + "max_run_cost_usd": 33, "timeout_seconds": 300, "tools_enabled": true } diff --git a/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json index 5c6783c..9d45a56 100644 --- a/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json +++ b/models/v4/frontier-2026-07/gemini-3-1-pro-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 8, + "max_run_cost_usd": 10, "timeout_seconds": 300, "tools_enabled": true } diff --git a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json index 97da8f1..7fab065 100644 --- a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json +++ b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 18, + "max_run_cost_usd": 17, "timeout_seconds": 300, "tools_enabled": true } diff --git a/src/evidencebench/runner_v4.py b/src/evidencebench/runner_v4.py index 1701105..f3be725 100644 --- a/src/evidencebench/runner_v4.py +++ b/src/evidencebench/runner_v4.py @@ -727,9 +727,14 @@ def evaluate(entry) -> dict: task_dir, task = entry task_output = destination / "workspaces" / task.id if task_output.exists(): - raise FileExistsError( - f"refusing to overwrite prior task workspace: {task_output}" - ) + interrupted_root = destination / "interrupted" + interrupted_root.mkdir(parents=True, exist_ok=True) + interrupted = interrupted_root / task.id + suffix = 1 + while interrupted.exists(): + interrupted = interrupted_root / f"{task.id}-{suffix}" + suffix += 1 + task_output.rename(interrupted) task_output.mkdir(parents=True) try: response, transcript, usage = _run_matter_task( From b9ce9eff2b626be4b6b2d25c9ecf161e1e6476b2 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 14:08:56 -0700 Subject: [PATCH 7/8] Balance v4 matter cost ceilings --- models/v4/frontier-2026-07/gpt-5-6-sol-matter.json | 2 +- models/v4/frontier-2026-07/grok-4-5-matter.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json index 7fab065..97da8f1 100644 --- a/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json +++ b/models/v4/frontier-2026-07/gpt-5-6-sol-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 17, + "max_run_cost_usd": 18, "timeout_seconds": 300, "tools_enabled": true } diff --git a/models/v4/frontier-2026-07/grok-4-5-matter.json b/models/v4/frontier-2026-07/grok-4-5-matter.json index 120cafa..70ab5ef 100644 --- a/models/v4/frontier-2026-07/grok-4-5-matter.json +++ b/models/v4/frontier-2026-07/grok-4-5-matter.json @@ -13,7 +13,7 @@ "max_output_tokens": 4000, "max_turns": 8, "concurrency": 2, - "max_run_cost_usd": 8, + "max_run_cost_usd": 7, "timeout_seconds": 300, "tools_enabled": true } From 2554f4592e1aa1bd0eae5340d66d771d9ca58cbe Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 14:51:20 -0700 Subject: [PATCH 8/8] Publish v4 frontier research snapshot --- .../aggregate-results.json | 512 ++++++++++++++++++ 1 file changed, 512 insertions(+) create mode 100644 results/v4-research-2026-07/aggregate-results.json diff --git a/results/v4-research-2026-07/aggregate-results.json b/results/v4-research-2026-07/aggregate-results.json new file mode 100644 index 0000000..7c7c743 --- /dev/null +++ b/results/v4-research-2026-07/aggregate-results.json @@ -0,0 +1,512 @@ +{ + "corpus": { + "combined_candidate_tree_sha256": "ed142ea92d19647c6fe064c63274e1085f33d743554ff791bd6855341bb67759", + "doctrine": { + "count": 240, + "families": 168, + "sha256": "5bef4dd64f13cdb6b046611cc339817d54161613e54885fcd67739b016d5cc9c" + }, + "matter": { + "count": 60, + "tree_sha256": "556b7049bd62098100132fde2be475680ec415b8d4f4a49082e75580bba567f4" + } + }, + "cost_accounting": { + "account_delta_usd": 90.83802740000002, + "full_run_reported_cost_usd": 75.89118115000001, + "hard_limit_usd": 100.0, + "smoke_reported_cost_usd": 1.10092825 + }, + "models": [ + { + "display_name": "Grok 4.5", + "doctrine": { + "authority_f1": 0.6795655695425432, + "by_domain": { + "D01_JUDICIAL_ADMINISTRATION": { + "doctrine_score": 0.929911530612245, + "family_count": 7 + }, + "D02_RELEVANCE_403": { + "doctrine_score": 0.8500843650793651, + "family_count": 12 + }, + "D03_CHARACTER_PROPENSITY_HABIT": { + "doctrine_score": 0.7433485578002245, + "family_count": 9 + }, + "D04_POLICY_EXCLUSIONS": { + "doctrine_score": 0.7674048346560847, + "family_count": 9 + }, + "D05_WITNESS_EXAMINATION": { + "doctrine_score": 0.6578250396825397, + "family_count": 10 + }, + "D06_IMPEACHMENT_REHABILITATION": { + "doctrine_score": 0.7184155291005292, + "family_count": 9 + }, + "D07_OPINION_EXPERTS": { + "doctrine_score": 0.6450634888888889, + "family_count": 50 + }, + "D08_HEARSAY_DEFINITIONS": { + "doctrine_score": 0.673928787878788, + "family_count": 11 + }, + "D09_HEARSAY_EXCEPTIONS": { + "doctrine_score": 0.7615786441798942, + "family_count": 12 + }, + "D10_AUTHENTICATION_IDENTIFICATION": { + "doctrine_score": 0.7567692418546367, + "family_count": 19 + }, + "D11_CONTENTS_ORIGINALS_SUMMARIES": { + "doctrine_score": 0.7701217460317461, + "family_count": 11 + }, + "D12_PRIVILEGE_CONSTITUTIONAL": { + "doctrine_score": 0.6659981150793651, + "family_count": 9 + } + }, + "calibration": 0.8016672849358244, + "doctrine_score": 0.7450374900703589, + "domain_count": 12, + "failure_count": 0, + "family_count": 168, + "family_mean_score": 0.7202406932419433, + "grounding_f1": 0.6527064019978598, + "hallucinated_authority_count": 7, + "invalid_authority_count": 83, + "issue_f1": 0.8081458143464723, + "item_mean_score": 0.7254067198472823, + "n": 240, + "outcome_accuracy": 0.7543347953216374, + "score_ci_95": [ + 0.7166296107282839, + 0.7727459603180721 + ] + }, + "matter": { + "authority_grounding_rate": 0.2794863145538315, + "complete_task_rate": 0, + "deliverable_completeness_rate": 0.9833333333333333, + "factual_accuracy_rate": 0.6985219135021467, + "failure_count": 1, + "hallucinated_authority_count": 68, + "invalid_authority_count": 210, + "legal_criteria_rate": 0.763888888888889, + "matter_score": 0.6901199943805327, + "n": 60, + "score_ci_95": [ + 0.6460994974404253, + 0.7319718358350322 + ] + }, + "model_id": "x-ai/grok-4.5", + "overall_ci_95": [ + 0.6905934562805046, + 0.7423452842979248 + ], + "overall_score": 0.7175787422254458, + "overall_score_100": 71.75787422254459, + "run_cost_usd": { + "doctrine": 1.6318896000000003, + "matter": 5.966928000000001, + "total": 7.598817600000001 + }, + "run_protocol": { + "doctrine_manifest": "grok-4-5-doctrine.json", + "matter_manifest": "grok-4-5-matter.json", + "provider": "OpenRouter", + "runs_per_model": 1 + }, + "schema_version": "4.0", + "slug": "grok-4-5", + "track_weights": { + "doctrine": 0.5, + "matter": 0.5 + }, + "weight_sensitivity": { + "doctrine_40_matter_60": 0.7120869926564632, + "doctrine_50_matter_50": 0.7175787422254458, + "doctrine_60_matter_40": 0.7230704917944284 + } + }, + { + "display_name": "Gemini 3.1 Pro Preview", + "doctrine": { + "authority_f1": 0.47163779397989924, + "by_domain": { + "D01_JUDICIAL_ADMINISTRATION": { + "doctrine_score": 0.8806998299319728, + "family_count": 7 + }, + "D02_RELEVANCE_403": { + "doctrine_score": 0.6986154067460317, + "family_count": 12 + }, + "D03_CHARACTER_PROPENSITY_HABIT": { + "doctrine_score": 0.6748091380070547, + "family_count": 9 + }, + "D04_POLICY_EXCLUSIONS": { + "doctrine_score": 0.5125708774250441, + "family_count": 9 + }, + "D05_WITNESS_EXAMINATION": { + "doctrine_score": 0.63753125, + "family_count": 10 + }, + "D06_IMPEACHMENT_REHABILITATION": { + "doctrine_score": 0.742932208994709, + "family_count": 9 + }, + "D07_OPINION_EXPERTS": { + "doctrine_score": 0.6481599107142857, + "family_count": 50 + }, + "D08_HEARSAY_DEFINITIONS": { + "doctrine_score": 0.46652083333333333, + "family_count": 11 + }, + "D09_HEARSAY_EXCEPTIONS": { + "doctrine_score": 0.6276372767857143, + "family_count": 12 + }, + "D10_AUTHENTICATION_IDENTIFICATION": { + "doctrine_score": 0.7215534931077694, + "family_count": 19 + }, + "D11_CONTENTS_ORIGINALS_SUMMARIES": { + "doctrine_score": 0.6615588474025974, + "family_count": 11 + }, + "D12_PRIVILEGE_CONSTITUTIONAL": { + "doctrine_score": 0.5366147486772487, + "family_count": 9 + } + }, + "calibration": 0.7244639491326169, + "doctrine_score": 0.6507669850938135, + "domain_count": 12, + "failure_count": 24, + "family_count": 168, + "family_mean_score": 0.6499046110874905, + "grounding_f1": 0.5452742929604354, + "hallucinated_authority_count": 6, + "invalid_authority_count": 87, + "issue_f1": 0.7709923061019552, + "item_mean_score": 0.6429650171957673, + "n": 240, + "outcome_accuracy": 0.682351807549176, + "score_ci_95": [ + 0.6039834827890556, + 0.6937416354851071 + ] + }, + "matter": { + "authority_grounding_rate": 0.31546232248319533, + "complete_task_rate": 0, + "deliverable_completeness_rate": 0.9916666666666667, + "factual_accuracy_rate": 0.5797376564719943, + "failure_count": 0, + "hallucinated_authority_count": 8, + "invalid_authority_count": 62, + "legal_criteria_rate": 0.6538888888888889, + "matter_score": 0.6257475574118827, + "n": 60, + "score_ci_95": [ + 0.5784793655511951, + 0.6703297356210125 + ] + }, + "model_id": "google/gemini-3.1-pro-preview", + "overall_ci_95": [ + 0.6063166315041517, + 0.6694431818182489 + ], + "overall_score": 0.638257271252848, + "overall_score_100": 63.8257271252848, + "run_cost_usd": { + "doctrine": 1.9768740000000002, + "matter": 9.361858800000004, + "total": 11.338732800000004 + }, + "run_protocol": { + "doctrine_manifest": "gemini-3-1-pro-doctrine.json", + "matter_manifest": "gemini-3-1-pro-matter.json", + "provider": "OpenRouter", + "runs_per_model": 1 + }, + "schema_version": "4.0", + "slug": "gemini-3-1-pro", + "track_weights": { + "doctrine": 0.5, + "matter": 0.5 + }, + "weight_sensitivity": { + "doctrine_40_matter_60": 0.635755328484655, + "doctrine_50_matter_50": 0.638257271252848, + "doctrine_60_matter_40": 0.6407592140210412 + } + }, + { + "display_name": "GPT-5.6 Sol", + "doctrine": { + "authority_f1": 0.3346230090166494, + "by_domain": { + "D01_JUDICIAL_ADMINISTRATION": { + "doctrine_score": 0.8158951020408164, + "family_count": 7 + }, + "D02_RELEVANCE_403": { + "doctrine_score": 0.7832546825396826, + "family_count": 12 + }, + "D03_CHARACTER_PROPENSITY_HABIT": { + "doctrine_score": 0.6363732623857624, + "family_count": 9 + }, + "D04_POLICY_EXCLUSIONS": { + "doctrine_score": 0.6231965211640211, + "family_count": 9 + }, + "D05_WITNESS_EXAMINATION": { + "doctrine_score": 0.5758413988095238, + "family_count": 10 + }, + "D06_IMPEACHMENT_REHABILITATION": { + "doctrine_score": 0.588886919592753, + "family_count": 9 + }, + "D07_OPINION_EXPERTS": { + "doctrine_score": 0.5328493507936508, + "family_count": 50 + }, + "D08_HEARSAY_DEFINITIONS": { + "doctrine_score": 0.5117575703463203, + "family_count": 11 + }, + "D09_HEARSAY_EXCEPTIONS": { + "doctrine_score": 0.665543253968254, + "family_count": 12 + }, + "D10_AUTHENTICATION_IDENTIFICATION": { + "doctrine_score": 0.6853305433090302, + "family_count": 19 + }, + "D11_CONTENTS_ORIGINALS_SUMMARIES": { + "doctrine_score": 0.4799148412698413, + "family_count": 11 + }, + "D12_PRIVILEGE_CONSTITUTIONAL": { + "doctrine_score": 0.46263275793650793, + "family_count": 9 + } + }, + "calibration": 0.7003783272923471, + "doctrine_score": 0.6134563503463469, + "domain_count": 12, + "failure_count": 15, + "family_count": 168, + "family_mean_score": 0.596590502860235, + "grounding_f1": 0.6487512021119216, + "hallucinated_authority_count": 46, + "invalid_authority_count": 149, + "issue_f1": 0.7614813812730479, + "item_mean_score": 0.6248859052128427, + "n": 240, + "outcome_accuracy": 0.640668416622364, + "score_ci_95": [ + 0.5735897830833143, + 0.6496880238084952 + ] + }, + "matter": { + "authority_grounding_rate": 0.2526370391642979, + "complete_task_rate": 0, + "deliverable_completeness_rate": 1.0, + "factual_accuracy_rate": 0.6269484859432993, + "failure_count": 0, + "hallucinated_authority_count": 55, + "invalid_authority_count": 181, + "legal_criteria_rate": 0.6561111111111111, + "matter_score": 0.6226252362799101, + "n": 60, + "score_ci_95": [ + 0.5755175709347533, + 0.6703065084953028 + ] + }, + "model_id": "openai/gpt-5.6-sol", + "overall_ci_95": [ + 0.5870945127740369, + 0.6485075711581395 + ], + "overall_score": 0.6180407933131284, + "overall_score_100": 61.80407933131284, + "run_cost_usd": { + "doctrine": 4.931025000000002, + "matter": 15.981245750000008, + "total": 20.912270750000012 + }, + "run_protocol": { + "doctrine_manifest": "gpt-5-6-sol-doctrine.json", + "matter_manifest": "gpt-5-6-sol-matter.json", + "provider": "OpenRouter", + "runs_per_model": 1 + }, + "schema_version": "4.0", + "slug": "gpt-5-6-sol", + "track_weights": { + "doctrine": 0.5, + "matter": 0.5 + }, + "weight_sensitivity": { + "doctrine_40_matter_60": 0.6189576819064848, + "doctrine_50_matter_50": 0.6180407933131284, + "doctrine_60_matter_40": 0.6171239047197722 + } + }, + { + "display_name": "Claude Opus 5", + "doctrine": { + "authority_f1": 0.303356948279489, + "by_domain": { + "D01_JUDICIAL_ADMINISTRATION": { + "doctrine_score": 0.8053521088435375, + "family_count": 7 + }, + "D02_RELEVANCE_403": { + "doctrine_score": 0.5946858730158731, + "family_count": 12 + }, + "D03_CHARACTER_PROPENSITY_HABIT": { + "doctrine_score": 0.5883989197530864, + "family_count": 9 + }, + "D04_POLICY_EXCLUSIONS": { + "doctrine_score": 0.5439361838624339, + "family_count": 9 + }, + "D05_WITNESS_EXAMINATION": { + "doctrine_score": 0.5378181904761905, + "family_count": 10 + }, + "D06_IMPEACHMENT_REHABILITATION": { + "doctrine_score": 0.5608593360189194, + "family_count": 9 + }, + "D07_OPINION_EXPERTS": { + "doctrine_score": 0.5296488670634921, + "family_count": 50 + }, + "D08_HEARSAY_DEFINITIONS": { + "doctrine_score": 0.445409025974026, + "family_count": 11 + }, + "D09_HEARSAY_EXCEPTIONS": { + "doctrine_score": 0.762276304563492, + "family_count": 12 + }, + "D10_AUTHENTICATION_IDENTIFICATION": { + "doctrine_score": 0.6773848527568923, + "family_count": 19 + }, + "D11_CONTENTS_ORIGINALS_SUMMARIES": { + "doctrine_score": 0.6413386507936508, + "family_count": 11 + }, + "D12_PRIVILEGE_CONSTITUTIONAL": { + "doctrine_score": 0.5174403240740741, + "family_count": 9 + } + }, + "calibration": 0.7793880232541581, + "doctrine_score": 0.600379053099639, + "domain_count": 12, + "failure_count": 0, + "family_count": 168, + "family_mean_score": 0.5863207480416409, + "grounding_f1": 0.5671399680885113, + "hallucinated_authority_count": 65, + "invalid_authority_count": 282, + "issue_f1": 0.7069871912525421, + "item_mean_score": 0.6122132091450216, + "n": 240, + "outcome_accuracy": 0.6681936691476165, + "score_ci_95": [ + 0.569082037370538, + 0.6319725487498834 + ] + }, + "matter": { + "authority_grounding_rate": 0.2730853544711694, + "complete_task_rate": 0, + "deliverable_completeness_rate": 0.9833333333333333, + "factual_accuracy_rate": 0.5561483503164383, + "failure_count": 1, + "hallucinated_authority_count": 42, + "invalid_authority_count": 184, + "legal_criteria_rate": 0.47444444444444445, + "matter_score": 0.5227615456639219, + "n": 60, + "score_ci_95": [ + 0.4601466936515833, + 0.5862691998991645 + ] + }, + "model_id": "anthropic/claude-opus-5", + "overall_ci_95": [ + 0.525630197319481, + 0.5970164821398064 + ], + "overall_score": 0.5615702993817804, + "overall_score_100": 56.15702993817804, + "run_cost_usd": { + "doctrine": 4.455399999999998, + "matter": 31.58595999999999, + "total": 36.04135999999999 + }, + "run_protocol": { + "doctrine_manifest": "claude-opus-5-doctrine.json", + "matter_manifest": "claude-opus-5-matter.json", + "provider": "OpenRouter", + "runs_per_model": 1 + }, + "schema_version": "4.0", + "slug": "claude-opus-5", + "track_weights": { + "doctrine": 0.5, + "matter": 0.5 + }, + "weight_sensitivity": { + "doctrine_40_matter_60": 0.5538085486382087, + "doctrine_50_matter_50": 0.5615702993817804, + "doctrine_60_matter_40": 0.5693320501253522 + } + } + ], + "protocol": { + "holdout_visibility": "private", + "models": 4, + "overall_score": "50% Doctrine + 50% Matter", + "provider": "OpenRouter", + "run_date": "2026-07-25", + "runs_per_model": 1 + }, + "published_as_official": false, + "release_id": "v4-research-2026-07", + "release_status": "unreviewed_research_release", + "review_status": { + "disclosure": "All gold labels are unreviewed construction candidates. Results are a transparent research release, not an attorney-validated official leaderboard.", + "professional_review_completed": false, + "reviewed_items": 0, + "total_items": 300 + }, + "schema_version": "4.0" +}