From 3b20f2ad9df73a0c8835f1839360cfa21d84c285 Mon Sep 17 00:00:00 2001 From: Jonathan Asare <294263306+jonathanjasare@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:35:17 +0100 Subject: [PATCH 1/2] feat: add provider-neutral conformance contract --- conformance/sequence-v1/events.jsonl | 37 +++ .../fixtures/window/tn/expired.jsonl | 3 + .../fixtures/window/tp/boundary.jsonl | 3 + conformance/sequence-v1/manifest.json | 17 ++ .../rules/TW-CONFORMANCE-WINDOW.yaml | 34 +++ docs/adr-002-backend-conformance.md | 47 +++ docs/decisions.md | 6 + results/conformance-sequence-v1.json | 214 +++++++++++++ src/tripwire/cli.py | 25 ++ src/tripwire/conformance.py | 289 ++++++++++++++++++ src/tripwire/core/rules.py | 42 ++- src/tripwire/evaluation.py | 5 +- tests/test_m3_conformance.py | 84 +++++ 13 files changed, 802 insertions(+), 4 deletions(-) create mode 100644 conformance/sequence-v1/events.jsonl create mode 100644 conformance/sequence-v1/fixtures/window/tn/expired.jsonl create mode 100644 conformance/sequence-v1/fixtures/window/tp/boundary.jsonl create mode 100644 conformance/sequence-v1/manifest.json create mode 100644 conformance/sequence-v1/rules/TW-CONFORMANCE-WINDOW.yaml create mode 100644 docs/adr-002-backend-conformance.md create mode 100644 results/conformance-sequence-v1.json create mode 100644 src/tripwire/conformance.py create mode 100644 tests/test_m3_conformance.py diff --git a/conformance/sequence-v1/events.jsonl b/conformance/sequence-v1/events.jsonl new file mode 100644 index 0000000..115268f --- /dev/null +++ b/conformance/sequence-v1/events.jsonl @@ -0,0 +1,37 @@ +{"session_id":"case-01-within-299","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-01-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-01-within-299","seq":1,"observed_at":"2026-08-11T12:00:10+00:00","kind":"agent_output","tool_name":null,"tool_role":null,"source_trust":null,"tool_call_id":null,"arguments":null,"result":"VALUE_002","outcome":null} +{"session_id":"case-01-within-299","seq":2,"observed_at":"2026-08-11T12:00:30+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-01-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-01-within-299","seq":3,"observed_at":"2026-08-11T12:01:00+00:00","kind":"tool_call","tool_name":"unrelated_tool","tool_role":"other","source_trust":null,"tool_call_id":"case-01-call-3","arguments":{},"result":null,"outcome":null} +{"session_id":"case-01-within-299","seq":4,"observed_at":"2026-08-11T12:04:59+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-01-call-4","arguments":{},"result":null,"outcome":null} +{"session_id":"case-02-boundary-300","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-02-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-02-boundary-300","seq":1,"observed_at":"2026-08-11T12:02:30+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-02-call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"case-02-boundary-300","seq":2,"observed_at":"2026-08-11T12:05:00+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-02-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-03-outside-301","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-03-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-03-outside-301","seq":1,"observed_at":"2026-08-11T12:02:30+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-03-call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"case-03-outside-301","seq":2,"observed_at":"2026-08-11T12:05:01+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-03-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-04-missing-time","seq":0,"observed_at":null,"kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-04-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-04-missing-time","seq":1,"observed_at":null,"kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-04-call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"case-04-missing-time","seq":2,"observed_at":null,"kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-04-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-05-wrong-order","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-05-call-0","arguments":{},"result":null,"outcome":null} +{"session_id":"case-05-wrong-order","seq":1,"observed_at":"2026-08-11T12:00:10+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-05-call-1","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-05-wrong-order","seq":2,"observed_at":"2026-08-11T12:00:20+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-05-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-06-multiple-candidates","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external_first","tool_role":"external_input","source_trust":"external","tool_call_id":"case-06-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-06-multiple-candidates","seq":1,"observed_at":"2026-08-11T12:00:10+00:00","kind":"tool_result","tool_name":"retrieve_external_second","tool_role":"external_input","source_trust":"external","tool_call_id":"case-06-call-1","arguments":{},"result":"VALUE_002","outcome":"success"} +{"session_id":"case-06-multiple-candidates","seq":2,"observed_at":"2026-08-11T12:00:20+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-06-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-06-multiple-candidates","seq":3,"observed_at":"2026-08-11T12:00:30+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-06-call-3","arguments":{},"result":null,"outcome":null} +{"session_id":"case-07-later-valid-source","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external_expired","tool_role":"external_input","source_trust":"external","tool_call_id":"case-07-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-07-later-valid-source","seq":1,"observed_at":"2026-08-11T12:15:00+00:00","kind":"tool_result","tool_name":"retrieve_external_current","tool_role":"external_input","source_trust":"external","tool_call_id":"case-07-call-1","arguments":{},"result":"VALUE_002","outcome":"success"} +{"session_id":"case-07-later-valid-source","seq":2,"observed_at":"2026-08-11T12:15:50+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-07-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-07-later-valid-source","seq":3,"observed_at":"2026-08-11T12:16:40+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-07-call-3","arguments":{},"result":null,"outcome":null} +{"session_id":"case-08-tied-time","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-08-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-08-tied-time","seq":1,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-08-call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"case-08-tied-time","seq":2,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-08-call-2","arguments":{},"result":null,"outcome":null} +{"session_id":"case-09-source-only","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-09-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-10-tail-only","seq":0,"observed_at":"2026-08-11T12:00:10+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-10-call-0","arguments":{},"result":null,"outcome":null} +{"session_id":"case-10-tail-only","seq":1,"observed_at":"2026-08-11T12:00:20+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-10-call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"case-11-gapped-seq","seq":10,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-11-call-10","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-11-gapped-seq","seq":20,"observed_at":"2026-08-11T12:00:30+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-11-call-20","arguments":{},"result":null,"outcome":null} +{"session_id":"case-11-gapped-seq","seq":30,"observed_at":"2026-08-11T12:01:00+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"case-11-call-30","arguments":{},"result":null,"outcome":null} +{"session_id":"case-12-no-external-write","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"case-12-call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"case-12-no-external-write","seq":1,"observed_at":"2026-08-11T12:00:30+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"case-12-call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"case-12-no-external-write","seq":2,"observed_at":"2026-08-11T12:01:00+00:00","kind":"tool_call","tool_name":"change_internal_state","tool_role":"state_change","source_trust":null,"tool_call_id":"case-12-call-2","arguments":{},"result":null,"outcome":null} diff --git a/conformance/sequence-v1/fixtures/window/tn/expired.jsonl b/conformance/sequence-v1/fixtures/window/tn/expired.jsonl new file mode 100644 index 0000000..ae42be2 --- /dev/null +++ b/conformance/sequence-v1/fixtures/window/tn/expired.jsonl @@ -0,0 +1,3 @@ +{"session_id":"fixture-window-expired","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"fixture-window-expired","seq":1,"observed_at":"2026-08-11T12:02:30+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"fixture-window-expired","seq":2,"observed_at":"2026-08-11T12:05:01+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"call-2","arguments":{},"result":null,"outcome":null} diff --git a/conformance/sequence-v1/fixtures/window/tp/boundary.jsonl b/conformance/sequence-v1/fixtures/window/tp/boundary.jsonl new file mode 100644 index 0000000..0055f71 --- /dev/null +++ b/conformance/sequence-v1/fixtures/window/tp/boundary.jsonl @@ -0,0 +1,3 @@ +{"session_id":"fixture-window-boundary","seq":0,"observed_at":"2026-08-11T12:00:00+00:00","kind":"tool_result","tool_name":"retrieve_external","tool_role":"external_input","source_trust":"external","tool_call_id":"call-0","arguments":{},"result":"VALUE_001","outcome":"success"} +{"session_id":"fixture-window-boundary","seq":1,"observed_at":"2026-08-11T12:02:30+00:00","kind":"tool_call","tool_name":"read_sensitive","tool_role":"sensitive_read","source_trust":null,"tool_call_id":"call-1","arguments":{},"result":null,"outcome":null} +{"session_id":"fixture-window-boundary","seq":2,"observed_at":"2026-08-11T12:05:00+00:00","kind":"tool_call","tool_name":"write_external","tool_role":"external_write","source_trust":null,"tool_call_id":"call-2","arguments":{},"result":null,"outcome":null} diff --git a/conformance/sequence-v1/manifest.json b/conformance/sequence-v1/manifest.json new file mode 100644 index 0000000..714bc26 --- /dev/null +++ b/conformance/sequence-v1/manifest.json @@ -0,0 +1,17 @@ +{ + "events_file": "conformance/sequence-v1/events.jsonl", + "events_sha256": "fabc944cfff284beb1da58f79eb6f861a2ec3e7b8fca75929e832aed34eb97b4", + "result_file": "results/conformance-sequence-v1.json", + "rules": [ + { + "path": "rules/TW-001.yaml", + "purpose": "measured_rule" + }, + { + "path": "conformance/sequence-v1/rules/TW-CONFORMANCE-WINDOW.yaml", + "purpose": "test_only_window_semantics" + } + ], + "schema_version": 1, + "suite_id": "sequence-v1" +} diff --git a/conformance/sequence-v1/rules/TW-CONFORMANCE-WINDOW.yaml b/conformance/sequence-v1/rules/TW-CONFORMANCE-WINDOW.yaml new file mode 100644 index 0000000..451588f --- /dev/null +++ b/conformance/sequence-v1/rules/TW-CONFORMANCE-WINDOW.yaml @@ -0,0 +1,34 @@ +schema_version: 1 +id: TW-CONFORMANCE-WINDOW +title: Test-only bounded sequence semantics +status: test +severity: informational + +scope: + corpus: sequence-v1 + +sequence: + - as: source + kind: tool_result + tool_role: external_input + source_trust: external + - as: sensitive + kind: tool_call + tool_role: sensitive_read + - as: sink + kind: tool_call + tool_role: external_write + +match: + group_by: [session_id] + order: [source, sensitive, sink] + within: 300 + +tests: + true_positives: + - conformance/sequence-v1/fixtures/window/tp/*.jsonl + true_negatives: + - conformance/sequence-v1/fixtures/window/tn/*.jsonl + +references: + - Provider-neutral timestamp-window conformance only; not a measured detection rule diff --git a/docs/adr-002-backend-conformance.md b/docs/adr-002-backend-conformance.md new file mode 100644 index 0000000..204308c --- /dev/null +++ b/docs/adr-002-backend-conformance.md @@ -0,0 +1,47 @@ +# ADR-002: Local-engine backend conformance contract + +## Status + +Accepted on 2026-08-11. + +## Context + +The measured `TW-001` result uses historic AgentDojo events without per-event timestamps. It +establishes ordering behaviour, but it cannot establish elapsed-time behaviour. A future provider +query also needs an exact reference result before its status can move from rendered or executed to +conformant. + +## Decision + +The local engine is the reference implementation. The fixed `sequence-v1` suite supplies 37 +synthetic, normalised events across 12 sessions and commits the exact local result for two rules: + +- the unchanged, measured `TW-001` rule for ordering parity; +- a test-only 300-second rule for timestamp-window parity. + +The test-only rule is not a published detector and carries no effectiveness or performance claim. +It exists outside `rules/` so that `TW-001` remains the only measured rule. + +Conformance is exact. A provider implementation must reproduce each rule identifier and hash, +session identifier, matched event ordinal and matched tool name. The suite also fixes these +semantics: + +- events are partitioned by session and ordered by non-negative `seq` values; +- sequence order is strict, while ordinal values may contain gaps; +- a window includes its exact boundary; +- missing timestamps fail a bounded match; +- candidate search may skip an expired earlier event to find a later valid sequence; +- events from different sessions cannot complete one sequence. + +Rules and suite inputs reject unknown fields. This prevents a misspelled field from silently +broadening a rendered query. + +No provider interface or placeholder backend is added now. The first real provider consumer will +define its mapping and rendering contract. Backend status remains per rule: `unsupported`, +`rendered`, `executed` or `conformant`. Only exact parity on this fixed suite permits `conformant`. + +## Consequences + +`uv run tripwire verify` now checks both the frozen AgentDojo measurement and the provider-neutral +conformance result without credentials. Provider-specific rendering, ingestion, execution, +alerting, latency and cost evidence remain later work. diff --git a/docs/decisions.md b/docs/decisions.md index a41c908..9ae94c9 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -17,3 +17,9 @@ The canonical split groups by `(suite, user task, injection task)`, so it isolat ## 2026-08-03: Naive-sequence stop-condition check The naive external-result then external-write diagnostic matched 4 of 74 development negatives and 1 of 37 holdout negatives. `TW-001` matched 3 of 74 and 1 of 37 respectively. The documented stop condition was not reached because `TW-001` has better specificity on the frozen development data, although the improvement did not repeat on holdout. This limitation is reported without changing the rule after holdout evaluation. + +## 2026-08-11: Freeze the provider-neutral conformance boundary before M3 + +The local engine is the reference for future provider implementations. The committed `sequence-v1` suite separately fixes ordering and elapsed-time semantics using synthetic normalised events, while the measured `TW-001` rule and its spent holdout remain unchanged. A test-only bounded rule lives outside the published rule directory and makes no detection-effectiveness claim. + +No generic backend interface is introduced before a real provider consumer exists. A provider earns `conformant` status per rule only after its executed result exactly matches the committed session identifiers, rule hashes, event ordinals and tool names. Provider rendering, ingestion, execution, alerting, latency and cost evidence begin after this boundary. diff --git a/results/conformance-sequence-v1.json b/results/conformance-sequence-v1.json new file mode 100644 index 0000000..2ed2362 --- /dev/null +++ b/results/conformance-sequence-v1.json @@ -0,0 +1,214 @@ +{ + "engine_version": "0.1.0", + "event_count": 37, + "events_sha256": "fabc944cfff284beb1da58f79eb6f861a2ec3e7b8fca75929e832aed34eb97b4", + "result_id": "sequence-v1", + "rules": [ + { + "detections": [ + { + "matched_event_ordinals": [ + 0, + 2, + 4 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-01-within-299" + }, + { + "matched_event_ordinals": [ + 0, + 1, + 2 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-02-boundary-300" + }, + { + "matched_event_ordinals": [ + 0, + 1, + 2 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-03-outside-301" + }, + { + "matched_event_ordinals": [ + 0, + 1, + 2 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-04-missing-time" + }, + { + "matched_event_ordinals": [ + 0, + 2, + 3 + ], + "matched_tool_names": [ + "retrieve_external_first", + "read_sensitive", + "write_external" + ], + "session_id": "case-06-multiple-candidates" + }, + { + "matched_event_ordinals": [ + 0, + 2, + 3 + ], + "matched_tool_names": [ + "retrieve_external_expired", + "read_sensitive", + "write_external" + ], + "session_id": "case-07-later-valid-source" + }, + { + "matched_event_ordinals": [ + 0, + 1, + 2 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-08-tied-time" + }, + { + "matched_event_ordinals": [ + 10, + 20, + 30 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-11-gapped-seq" + } + ], + "fixtures": { + "true_negatives": 2, + "true_positives": 1 + }, + "purpose": "measured_rule", + "rule_id": "TW-001", + "rule_sha256": "9304bec9d515024fff0efa833923866060d806b0d40ef02e14b92e195d6cc4e9" + }, + { + "detections": [ + { + "matched_event_ordinals": [ + 0, + 2, + 4 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-01-within-299" + }, + { + "matched_event_ordinals": [ + 0, + 1, + 2 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-02-boundary-300" + }, + { + "matched_event_ordinals": [ + 0, + 2, + 3 + ], + "matched_tool_names": [ + "retrieve_external_first", + "read_sensitive", + "write_external" + ], + "session_id": "case-06-multiple-candidates" + }, + { + "matched_event_ordinals": [ + 1, + 2, + 3 + ], + "matched_tool_names": [ + "retrieve_external_current", + "read_sensitive", + "write_external" + ], + "session_id": "case-07-later-valid-source" + }, + { + "matched_event_ordinals": [ + 0, + 1, + 2 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-08-tied-time" + }, + { + "matched_event_ordinals": [ + 10, + 20, + 30 + ], + "matched_tool_names": [ + "retrieve_external", + "read_sensitive", + "write_external" + ], + "session_id": "case-11-gapped-seq" + } + ], + "fixtures": { + "true_negatives": 1, + "true_positives": 1 + }, + "purpose": "test_only_window_semantics", + "rule_id": "TW-CONFORMANCE-WINDOW", + "rule_sha256": "3a5cfb552b80e2c4e857ddd476b6b39686a6d42d3ec138f45c20501fe54109f8" + } + ], + "schema_version": 1, + "session_count": 12 +} diff --git a/src/tripwire/cli.py b/src/tripwire/cli.py index 6270f3a..d9e30b9 100644 --- a/src/tripwire/cli.py +++ b/src/tripwire/cli.py @@ -6,6 +6,12 @@ from pathlib import Path from tripwire.adapters.agentdojo import load_taxonomy +from tripwire.conformance import ( + conformance_result_path, + evaluate_conformance, + verify_conformance, + write_conformance_result, +) from tripwire.corpus import build_corpus, inventory from tripwire.evaluation import evaluate_corpus, run_rule_tests, verify_repository, write_result @@ -38,6 +44,14 @@ def _parser() -> argparse.ArgumentParser: ) evaluate_parser.add_argument("--output", type=Path, required=True) + conformance = subcommands.add_parser("conformance") + conformance_subcommands = conformance.add_subparsers(dest="conformance_command", required=True) + conformance_evaluate = conformance_subcommands.add_parser("evaluate") + conformance_evaluate.add_argument("--suite", type=Path, required=True) + conformance_evaluate.add_argument("--output", type=Path) + conformance_verify = conformance_subcommands.add_parser("verify") + conformance_verify.add_argument("--suite", type=Path, required=True) + subcommands.add_parser("verify") report = subcommands.add_parser("report") report_subcommands = report.add_subparsers(dest="report_command", required=True) @@ -100,6 +114,17 @@ def main(argv: Sequence[str] | None = None) -> int: write_result(evaluation_result, args.output) print(f"Wrote reproducible result to {args.output}") return 0 + if args.command == "conformance": + root = Path.cwd() + if args.conformance_command == "evaluate": + result = evaluate_conformance(root, args.suite) + output = args.output or conformance_result_path(root, args.suite) + write_conformance_result(result, output) + print(f"Wrote reproducible conformance result to {output}") + return 0 + if args.conformance_command == "verify": + print(verify_conformance(root, args.suite)) + return 0 if args.command == "verify": print(verify_repository(Path.cwd())) return 0 diff --git a/src/tripwire/conformance.py b/src/tripwire/conformance.py new file mode 100644 index 0000000..9e281a1 --- /dev/null +++ b/src/tripwire/conformance.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Any, cast + +from tripwire import __version__ +from tripwire.adapters.agentdojo import canonical_json +from tripwire.core.engine import evaluate +from tripwire.core.events import AgentEvent +from tripwire.core.rules import load_rule +from tripwire.corpus import deserialise_event + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_EVENT_FIELDS = { + "session_id", + "seq", + "observed_at", + "kind", + "tool_name", + "tool_role", + "source_trust", + "tool_call_id", + "arguments", + "result", + "outcome", +} +_EVENT_KINDS = {"user_input", "agent_output", "tool_call", "tool_result"} +_TOOL_ROLES = { + "external_input", + "sensitive_read", + "external_write", + "state_change", + "other", + "unknown", +} +_SOURCE_TRUST = {"external", "internal", "unknown"} +_OUTCOMES = {"success", "failure", "refused", "unknown"} +_RULE_PURPOSES = {"measured_rule", "test_only_window_semantics"} + + +class ConformanceError(RuntimeError): + pass + + +def evaluate_conformance(root: Path, suite: Path) -> dict[str, Any]: + from tripwire.evaluation import run_rule_tests + + root = root.resolve() + suite_root = _resolve_path(root, suite, label="suite") + manifest_path = suite_root / "manifest.json" + manifest = _load_object(manifest_path) + _require_exact_keys( + manifest, + { + "schema_version", + "suite_id", + "events_file", + "events_sha256", + "result_file", + "rules", + }, + manifest_path, + "manifest", + ) + if manifest["schema_version"] != 1: + raise ConformanceError(f"{manifest_path}: unsupported schema_version") + suite_id = _non_empty_string(manifest, "suite_id", manifest_path) + expected_events_sha256 = _sha256(manifest, "events_sha256", manifest_path) + events_path = _resolve_path( + root, Path(_non_empty_string(manifest, "events_file", manifest_path)), label="events" + ) + try: + events_bytes = events_path.read_bytes() + except OSError as exc: + raise ConformanceError(f"{events_path}: cannot load event corpus: {exc}") from exc + actual_events_sha256 = hashlib.sha256(events_bytes).hexdigest() + if actual_events_sha256 != expected_events_sha256: + raise ConformanceError(f"{events_path}: event corpus hash mismatch") + sessions, event_count = _load_sessions(events_path, events_bytes) + + rules_raw = manifest["rules"] + if not isinstance(rules_raw, list) or not rules_raw: + raise ConformanceError(f"{manifest_path}: rules must be a non-empty array") + rule_results: list[dict[str, Any]] = [] + seen_rule_ids: set[str] = set() + for index, entry in enumerate(rules_raw): + if not isinstance(entry, dict): + raise ConformanceError(f"{manifest_path}: rule {index} must be an object") + _require_exact_keys(entry, {"path", "purpose"}, manifest_path, f"rule {index}") + rule_path = _resolve_path( + root, + Path(_non_empty_string(entry, "path", manifest_path)), + label=f"rule {index}", + ) + purpose = _non_empty_string(entry, "purpose", manifest_path) + if purpose not in _RULE_PURPOSES: + raise ConformanceError(f"{manifest_path}: rule {index} has invalid purpose") + rule = load_rule(rule_path) + if rule.rule_id in seen_rule_ids: + raise ConformanceError(f"{manifest_path}: duplicate rule id {rule.rule_id}") + seen_rule_ids.add(rule.rule_id) + + positive_fixtures, negative_fixtures = run_rule_tests(rule, root=root) + + detections: list[dict[str, Any]] = [] + for session_id, events in sorted(sessions.items()): + for match in evaluate(rule, events): + detections.append( + { + "matched_event_ordinals": list(match.matched_event_ordinals), + "matched_tool_names": list(match.matched_tool_names), + "session_id": session_id, + } + ) + rule_results.append( + { + "detections": detections, + "fixtures": { + "true_negatives": negative_fixtures, + "true_positives": positive_fixtures, + }, + "purpose": purpose, + "rule_id": rule.rule_id, + "rule_sha256": rule.sha256, + } + ) + + return { + "engine_version": __version__, + "event_count": event_count, + "events_sha256": actual_events_sha256, + "result_id": suite_id, + "rules": sorted(rule_results, key=lambda value: cast(str, value["rule_id"])), + "schema_version": 1, + "session_count": len(sessions), + } + + +def write_conformance_result(result: dict[str, Any], output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def verify_conformance(root: Path, suite: Path) -> str: + root = root.resolve() + suite_root = _resolve_path(root, suite, label="suite") + manifest_path = suite_root / "manifest.json" + manifest = _load_object(manifest_path) + result_path = _resolve_path( + root, + Path(_non_empty_string(manifest, "result_file", manifest_path)), + label="result", + ) + generated = evaluate_conformance(root, suite_root) + committed = _load_object(result_path) + if canonical_json(generated) != canonical_json(committed): + raise ConformanceError(f"{result_path}: committed conformance result does not reproduce") + detections = sum(len(cast(list[Any], rule["detections"])) for rule in generated["rules"]) + return ( + f"{generated['result_id']} verified: rules={len(generated['rules'])} " + f"sessions={generated['session_count']} detections={detections}" + ) + + +def conformance_result_path(root: Path, suite: Path) -> Path: + root = root.resolve() + suite_root = _resolve_path(root, suite, label="suite") + manifest_path = suite_root / "manifest.json" + manifest = _load_object(manifest_path) + return _resolve_path( + root, + Path(_non_empty_string(manifest, "result_file", manifest_path)), + label="result", + ) + + +def _load_sessions(path: Path, payload: bytes) -> tuple[dict[str, tuple[AgentEvent, ...]], int]: + grouped: dict[str, list[AgentEvent]] = {} + seen_ordinals: set[tuple[str, int]] = set() + event_count = 0 + try: + text = payload.decode("utf-8") + except UnicodeError as exc: + raise ConformanceError(f"{path}: events must be UTF-8") from exc + for line_number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + raw = json.loads(line) + except json.JSONDecodeError as exc: + raise ConformanceError(f"{path}:{line_number}: invalid JSON") from exc + event = _parse_event(raw, path, line_number) + identity = (event.session_id, event.seq) + if identity in seen_ordinals: + raise ConformanceError( + f"{path}:{line_number}: duplicate session ordinal {event.session_id}/{event.seq}" + ) + seen_ordinals.add(identity) + grouped.setdefault(event.session_id, []).append(event) + event_count += 1 + if not grouped: + raise ConformanceError(f"{path}: event corpus is empty") + sessions = { + session_id: tuple(sorted(events, key=lambda event: event.seq)) + for session_id, events in grouped.items() + } + return sessions, event_count + + +def _parse_event(raw: Any, path: Path, line_number: int) -> AgentEvent: + if not isinstance(raw, dict): + raise ConformanceError(f"{path}:{line_number}: event must be an object") + _require_exact_keys(raw, _EVENT_FIELDS, path, f"line {line_number}") + if not isinstance(raw["kind"], str) or raw["kind"] not in _EVENT_KINDS: + raise ConformanceError(f"{path}:{line_number}: invalid event kind") + if raw["tool_role"] is not None and ( + not isinstance(raw["tool_role"], str) or raw["tool_role"] not in _TOOL_ROLES + ): + raise ConformanceError(f"{path}:{line_number}: invalid tool role") + if raw["source_trust"] is not None and ( + not isinstance(raw["source_trust"], str) or raw["source_trust"] not in _SOURCE_TRUST + ): + raise ConformanceError(f"{path}:{line_number}: invalid source trust") + if raw["outcome"] is not None and ( + not isinstance(raw["outcome"], str) or raw["outcome"] not in _OUTCOMES + ): + raise ConformanceError(f"{path}:{line_number}: invalid outcome") + if not isinstance(raw["session_id"], str) or not raw["session_id"]: + raise ConformanceError(f"{path}:{line_number}: session_id must be non-empty") + if not isinstance(raw["seq"], int) or isinstance(raw["seq"], bool) or raw["seq"] < 0: + raise ConformanceError(f"{path}:{line_number}: seq must be a non-negative integer") + for key in ("tool_name", "tool_call_id"): + if raw[key] is not None and not isinstance(raw[key], str): + raise ConformanceError(f"{path}:{line_number}: {key} must be a string or null") + if raw["arguments"] is not None and not isinstance(raw["arguments"], dict): + raise ConformanceError(f"{path}:{line_number}: arguments must be an object or null") + try: + event = deserialise_event(raw, path) + except (TypeError, ValueError) as exc: + raise ConformanceError(f"{path}:{line_number}: invalid event: {exc}") from exc + if event.observed_at is not None and event.observed_at.utcoffset() is None: + raise ConformanceError(f"{path}:{line_number}: observed_at must include a timezone") + return event + + +def _load_object(path: Path) -> dict[str, Any]: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ConformanceError(f"{path}: cannot load JSON: {exc}") from exc + if not isinstance(raw, dict): + raise ConformanceError(f"{path}: root must be an object") + return raw + + +def _resolve_path(root: Path, path: Path, *, label: str) -> Path: + candidate = path.resolve() if path.is_absolute() else (root / path).resolve() + if not candidate.is_relative_to(root): + raise ConformanceError(f"{label} path escapes repository root: {path}") + return candidate + + +def _require_exact_keys(raw: dict[str, Any], expected: set[str], path: Path, label: str) -> None: + missing = sorted(expected - set(raw)) + unknown = sorted(set(raw) - expected) + if missing or unknown: + details: list[str] = [] + if missing: + details.append(f"missing {', '.join(missing)}") + if unknown: + details.append(f"unknown {', '.join(unknown)}") + raise ConformanceError(f"{path}: {label} fields invalid: {'; '.join(details)}") + + +def _non_empty_string(raw: dict[str, Any], key: str, path: Path) -> str: + value = raw.get(key) + if not isinstance(value, str) or not value: + raise ConformanceError(f"{path}: {key} must be a non-empty string") + return value + + +def _sha256(raw: dict[str, Any], key: str, path: Path) -> str: + value = _non_empty_string(raw, key, path) + if not _SHA256.fullmatch(value): + raise ConformanceError(f"{path}: {key} must be a lowercase SHA-256") + return value diff --git a/src/tripwire/core/rules.py b/src/tripwire/core/rules.py index 5dddf78..1b4a080 100644 --- a/src/tripwire/core/rules.py +++ b/src/tripwire/core/rules.py @@ -51,6 +51,23 @@ def load_rule(path: Path) -> Rule: raise RuleError(f"{path}: cannot load rule: {exc}") from exc if not isinstance(raw, dict): raise RuleError(f"{path}: rule root must be an object") + _reject_unknown_keys( + raw, + { + "schema_version", + "id", + "title", + "status", + "severity", + "scope", + "sequence", + "match", + "tests", + "references", + }, + path, + "rule", + ) sequence_raw = raw.get("sequence") if not isinstance(sequence_raw, list) or len(sequence_raw) < 2: raise RuleError(f"{path}: sequence must contain at least two steps") @@ -60,6 +77,7 @@ def load_rule(path: Path) -> Rule: raise RuleError(f"{path}: sequence aliases must be unique") match = _object(raw, "match", path) + _reject_unknown_keys(match, {"group_by", "order", "within"}, path, "match") group_by_raw = match.get("group_by") order_raw = match.get("order") if group_by_raw != ["session_id"]: @@ -73,11 +91,19 @@ def load_rule(path: Path) -> Rule: raise RuleError(f"{path}: within must be null or a positive integer number of seconds") tests_raw = _object(raw, "tests", path) + _reject_unknown_keys(tests_raw, {"true_positives", "true_negatives"}, path, "tests") tp = _string_list(tests_raw, "true_positives", path) tn = _string_list(tests_raw, "true_negatives", path) if not tp or not tn: raise RuleError(f"{path}: rule must own positive and negative fixtures") scope = _object(raw, "scope", path) + _reject_unknown_keys(scope, {"corpus"}, path, "scope") + references = raw.get("references") + if references is not None and ( + not isinstance(references, list) + or not all(isinstance(reference, str) and reference for reference in references) + ): + raise RuleError(f"{path}: references must be an array of non-empty strings") schema_version = _integer(raw, "schema_version", path) if schema_version != 1: raise RuleError(f"{path}: unsupported schema_version {schema_version}") @@ -99,6 +125,9 @@ def load_rule(path: Path) -> Rule: def _parse_step(path: Path, index: int, value: Any) -> SequenceStep: if not isinstance(value, dict): raise RuleError(f"{path}: sequence step {index} must be an object") + _reject_unknown_keys( + value, {"as", "kind", "tool_role", "source_trust"}, path, f"sequence step {index}" + ) valid_kinds = {"user_input", "agent_output", "tool_call", "tool_result"} valid_roles = { "external_input", @@ -112,11 +141,11 @@ def _parse_step(path: Path, index: int, value: Any) -> SequenceStep: kind = value.get("kind") role = value.get("tool_role") trust = value.get("source_trust") - if kind not in valid_kinds: + if not isinstance(kind, str) or kind not in valid_kinds: raise RuleError(f"{path}: sequence step {index} has invalid kind") - if role is not None and role not in valid_roles: + if role is not None and (not isinstance(role, str) or role not in valid_roles): raise RuleError(f"{path}: sequence step {index} has invalid tool_role") - if trust is not None and trust not in valid_trust: + if trust is not None and (not isinstance(trust, str) or trust not in valid_trust): raise RuleError(f"{path}: sequence step {index} has invalid source_trust") return SequenceStep( alias=_string(value, "as", path), @@ -152,3 +181,10 @@ def _string_list(raw: dict[str, Any], key: str, path: Path) -> tuple[str, ...]: if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value): raise RuleError(f"{path}: {key} must be an array of strings") return tuple(value) + + +def _reject_unknown_keys(raw: dict[str, Any], allowed: set[str], path: Path, label: str) -> None: + unknown = sorted(set(raw) - allowed, key=str) + if unknown: + names = ", ".join(str(field) for field in unknown) + raise RuleError(f"{path}: {label} contains unknown fields: {names}") diff --git a/src/tripwire/evaluation.py b/src/tripwire/evaluation.py index cf81e80..10771f5 100644 --- a/src/tripwire/evaluation.py +++ b/src/tripwire/evaluation.py @@ -101,6 +101,7 @@ def write_result(result: dict[str, Any], output: Path) -> None: def verify_repository(root: Path) -> str: + from tripwire.conformance import verify_conformance from tripwire.core.rules import load_rule from tripwire.reporting import render_readme @@ -121,12 +122,14 @@ def verify_repository(root: Path) -> str: raise VerificationError("README result summary was not generated from the canonical result") holdout = cast(dict[str, Any], generated["holdout"]) counts = cast(dict[str, Any], holdout["counts"]) - return ( + measured_summary = ( f"TW-001 verified: holdout TP={counts['true_positives']} FP={counts['false_positives']} " f"TN={counts['true_negatives']} FN={counts['false_negatives']}; " f"fixtures={positives} TP/{negatives} TN; " f"matched={len(cast(list[Any], holdout['matches']))}" ) + conformance_summary = verify_conformance(root, root / "conformance" / "sequence-v1") + return f"{measured_summary}; {conformance_summary}" def _evaluate_split(rule: Rule, runs: list[NormalisedRun]) -> dict[str, Any]: diff --git a/tests/test_m3_conformance.py b/tests/test_m3_conformance.py new file mode 100644 index 0000000..fbc20d2 --- /dev/null +++ b/tests/test_m3_conformance.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import Any, cast + +import pytest + +from tripwire.conformance import ConformanceError, evaluate_conformance, verify_conformance +from tripwire.core.rules import RuleError, load_rule + +ROOT = Path(__file__).parents[1] +SUITE = ROOT / "conformance" / "sequence-v1" + + +def test_provider_neutral_conformance_result_reproduces() -> None: + summary = verify_conformance(ROOT, SUITE) + result = evaluate_conformance(ROOT, SUITE) + + assert summary == "sequence-v1 verified: rules=2 sessions=12 detections=14" + assert result["event_count"] == 37 + assert result["session_count"] == 12 + assert sum(len(rule["detections"]) for rule in result["rules"]) == 14 + assert { + rule["rule_id"]: rule["fixtures"] for rule in cast(list[dict[str, Any]], result["rules"]) + } == { + "TW-001": {"true_negatives": 2, "true_positives": 1}, + "TW-CONFORMANCE-WINDOW": {"true_negatives": 1, "true_positives": 1}, + } + + +def test_conformance_covers_ordering_window_and_session_semantics() -> None: + result = evaluate_conformance(ROOT, SUITE) + detections = { + rule["rule_id"]: { + detection["session_id"]: detection["matched_event_ordinals"] + for detection in rule["detections"] + } + for rule in cast(list[dict[str, Any]], result["rules"]) + } + unbounded = detections["TW-001"] + bounded = detections["TW-CONFORMANCE-WINDOW"] + + assert "case-03-outside-301" in unbounded + assert "case-03-outside-301" not in bounded + assert "case-04-missing-time" in unbounded + assert "case-04-missing-time" not in bounded + assert bounded["case-02-boundary-300"] == [0, 1, 2] + assert unbounded["case-07-later-valid-source"] == [0, 2, 3] + assert bounded["case-07-later-valid-source"] == [1, 2, 3] + assert bounded["case-11-gapped-seq"] == [10, 20, 30] + assert "case-05-wrong-order" not in unbounded + assert "case-09-source-only" not in unbounded + assert "case-10-tail-only" not in unbounded + + +def test_window_rule_is_test_only_and_not_a_published_detector() -> None: + assert [path.name for path in sorted((ROOT / "rules").glob("*.yaml"))] == ["TW-001.yaml"] + rule = load_rule(SUITE / "rules" / "TW-CONFORMANCE-WINDOW.yaml") + assert rule.status == "test" + assert rule.severity == "informational" + assert rule.within_seconds == 300 + + +def test_conformance_event_corpus_hash_is_enforced(tmp_path: Path) -> None: + copied = tmp_path / "conformance" + shutil.copytree(ROOT / "conformance", copied) + events = copied / "sequence-v1" / "events.jsonl" + events.write_text(events.read_text(encoding="utf-8") + "\n", encoding="utf-8") + + with pytest.raises(ConformanceError, match="event corpus hash mismatch"): + evaluate_conformance(tmp_path, copied / "sequence-v1") + + +def test_unknown_rule_fields_fail_closed(tmp_path: Path) -> None: + source = (ROOT / "rules" / "TW-001.yaml").read_text(encoding="utf-8") + rule_path = tmp_path / "rule.yaml" + rule_path.write_text( + source.replace("source_trust: external", "source_trsut: external", 1), + encoding="utf-8", + ) + + with pytest.raises(RuleError, match="contains unknown fields: source_trsut"): + load_rule(rule_path) From 1551868be3821786a97678dea7d81cac1cc91088 Mon Sep 17 00:00:00 2001 From: Jonathan Asare <294263306+jonathanjasare@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:13:11 +0100 Subject: [PATCH 2/2] fix: reject malformed conformance metadata --- docs/adr-002-backend-conformance.md | 4 ++-- src/tripwire/conformance.py | 9 +++++++- tests/test_m3_conformance.py | 34 +++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docs/adr-002-backend-conformance.md b/docs/adr-002-backend-conformance.md index 204308c..9ed4a33 100644 --- a/docs/adr-002-backend-conformance.md +++ b/docs/adr-002-backend-conformance.md @@ -33,8 +33,8 @@ semantics: - candidate search may skip an expired earlier event to find a later valid sequence; - events from different sessions cannot complete one sequence. -Rules and suite inputs reject unknown fields. This prevents a misspelled field from silently -broadening a rendered query. +Rules and suite inputs reject unknown or malformed fields. This prevents a misspelled or wrongly +typed field from silently broadening a rendered query or changing timestamp-window semantics. No provider interface or placeholder backend is added now. The first real provider consumer will define its mapping and rendering contract. Backend status remains per rule: `unsupported`, diff --git a/src/tripwire/conformance.py b/src/tripwire/conformance.py index 9e281a1..cb31ec7 100644 --- a/src/tripwire/conformance.py +++ b/src/tripwire/conformance.py @@ -65,7 +65,12 @@ def evaluate_conformance(root: Path, suite: Path) -> dict[str, Any]: manifest_path, "manifest", ) - if manifest["schema_version"] != 1: + schema_version = manifest["schema_version"] + if ( + not isinstance(schema_version, int) + or isinstance(schema_version, bool) + or schema_version != 1 + ): raise ConformanceError(f"{manifest_path}: unsupported schema_version") suite_id = _non_empty_string(manifest, "suite_id", manifest_path) expected_events_sha256 = _sha256(manifest, "events_sha256", manifest_path) @@ -232,6 +237,8 @@ def _parse_event(raw: Any, path: Path, line_number: int) -> AgentEvent: raise ConformanceError(f"{path}:{line_number}: session_id must be non-empty") if not isinstance(raw["seq"], int) or isinstance(raw["seq"], bool) or raw["seq"] < 0: raise ConformanceError(f"{path}:{line_number}: seq must be a non-negative integer") + if raw["observed_at"] is not None and not isinstance(raw["observed_at"], str): + raise ConformanceError(f"{path}:{line_number}: observed_at must be a string or null") for key in ("tool_name", "tool_call_id"): if raw[key] is not None and not isinstance(raw[key], str): raise ConformanceError(f"{path}:{line_number}: {key} must be a string or null") diff --git a/tests/test_m3_conformance.py b/tests/test_m3_conformance.py index fbc20d2..d44b7d4 100644 --- a/tests/test_m3_conformance.py +++ b/tests/test_m3_conformance.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import json import shutil from pathlib import Path from typing import Any, cast @@ -72,6 +74,38 @@ def test_conformance_event_corpus_hash_is_enforced(tmp_path: Path) -> None: evaluate_conformance(tmp_path, copied / "sequence-v1") +def test_non_string_timestamp_fails_closed(tmp_path: Path) -> None: + copied = tmp_path / "conformance" + shutil.copytree(ROOT / "conformance", copied) + suite = copied / "sequence-v1" + events_path = suite / "events.jsonl" + lines = events_path.read_text(encoding="utf-8").splitlines() + first_event = json.loads(lines[0]) + first_event["observed_at"] = 0 + lines[0] = json.dumps(first_event, separators=(",", ":")) + events = ("\n".join(lines) + "\n").encode() + events_path.write_bytes(events) + + manifest_path = suite / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["events_sha256"] = hashlib.sha256(events).hexdigest() + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ConformanceError, match="observed_at must be a string or null"): + evaluate_conformance(tmp_path, suite) + + +def test_boolean_manifest_schema_version_fails_closed(tmp_path: Path) -> None: + suite = tmp_path / "conformance" / "sequence-v1" + suite.mkdir(parents=True) + manifest = json.loads((SUITE / "manifest.json").read_text(encoding="utf-8")) + manifest["schema_version"] = True + (suite / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ConformanceError, match="unsupported schema_version"): + evaluate_conformance(tmp_path, suite) + + def test_unknown_rule_fields_fail_closed(tmp_path: Path) -> None: source = (ROOT / "rules" / "TW-001.yaml").read_text(encoding="utf-8") rule_path = tmp_path / "rule.yaml"