From 4d0b8e60da4f68ac6bd153148c72bae81eb28fd5 Mon Sep 17 00:00:00 2001 From: Ashay Date: Sun, 9 Aug 2026 10:24:52 +0530 Subject: [PATCH] Add tamper example, quickstart output, and unknown-source edge test Walk each verification failure mode in examples/tamper.py (sources, grounding, merkle, re-sign with a different key) and paste representative output into the README. Improve the quickstart with install + expected verification output. Cover the untested case where source_contents maps an id the receipt never recorded. Fixes #1 Fixes #2 Fixes #4 --- README.md | 80 ++++++++++++++++++++++++++ examples/tamper.py | 127 +++++++++++++++++++++++++++++++++++++++++ tests/test_verifier.py | 18 ++++++ 3 files changed, 225 insertions(+) create mode 100644 examples/tamper.py diff --git a/README.md b/README.md index 3994827..df626c1 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ pip install "answerproof[api]" # + FastAPI verifier service ## Quickstart +```bash +pip install answerproof +``` + ```python from answerproof import ReceiptBuilder, SigningKey, verify_receipt @@ -86,6 +90,21 @@ verdict = verify_receipt( source_contents={"doc-1": "The Eiffel Tower is 330 metres tall."}, ) assert verdict.valid +print("valid:", verdict.valid) +for check in verdict.checks: + mark = "ok" if check.passed else "FAIL" + detail = f" — {check.detail}" if check.detail else "" + print(f" [{mark}] {check.name}{detail}") +``` + +Expected verification output (ids, timestamps, and digests will differ each run): + +```text +valid: True + [ok] signature + [ok] merkle + [ok] grounding — citations and grounding are self-consistent + [ok] sources — 1 source content(s) matched ``` ## Receipt anatomy @@ -147,6 +166,66 @@ Every check is independent and reported separately: | `grounding` | citations reference real sources; grounding score is honest | | `signer_pin`| (optional) signer public key matches an expected key | +### What failure looks like + +`examples/tamper.py` builds a valid receipt, then walks each failure mode and +prints the verdict. Run it with: + +```bash +python examples/tamper.py +``` + +Representative output (keys and digests vary per run): + +```text +=== Genuine receipt === +valid: True + [ok] signature + [ok] merkle + [ok] grounding — citations and grounding are self-consistent + [ok] sources — 3 source content(s) matched + +=== 1. Source document edited after signing === +valid: False + [ok] signature + [ok] merkle + [ok] grounding — citations and grounding are self-consistent + [FAIL] sources — content hash mismatch: s1 + +=== 2. Citation swapped to a different source id === +valid: False + [FAIL] signature — Ed25519 signature does not match payload + [ok] merkle + [FAIL] grounding — citation references unknown source not-a-real-source + [ok] sources — 3 source content(s) matched + +=== 3. Merkle root edited by hand === +valid: False + [FAIL] signature — Ed25519 signature does not match payload + [FAIL] merkle — recomputed Merkle root does not match signed root + [ok] grounding — citations and grounding are self-consistent + [ok] sources — 3 source content(s) matched + +=== 4a. Re-signed with attacker's key (no pin) === +valid: True + [ok] signature + [ok] merkle + [ok] grounding — citations and grounding are self-consistent + [ok] sources — 3 source content(s) matched + +=== 4b. Same receipt, signer pinned to the original key === +valid: False + [FAIL] signer_pin — receipt public key does not match expected signer + [ok] signature + [ok] merkle + [ok] grounding — citations and grounding are self-consistent + [ok] sources — 3 source content(s) matched +``` + +Case 4 is the interesting one: re-signing with a different key makes the +signature check pass again. Only pinning the expected public key (`signer_pin`) +catches the substitution — integrity alone is not provenance. + ## Merkle inclusion proofs Prove one source was in the retrieval set without revealing the rest: @@ -218,6 +297,7 @@ cd answerproof pip install -e ".[dev]" pytest -q # full test suite python examples/demo_rag.py # produce + verify a receipt, then tamper and re-verify +python examples/tamper.py # walk each failure mode (sources, grounding, merkle, re-sign) ``` ## Contributing diff --git a/examples/tamper.py b/examples/tamper.py new file mode 100644 index 0000000..9cf63eb --- /dev/null +++ b/examples/tamper.py @@ -0,0 +1,127 @@ +"""Walk each verification failure mode so newcomers can see what a bad receipt looks like. + +Run with no external services: + + python examples/tamper.py + +Pairs with ``tests/test_tamper.py``: same mutations, printed as a demo rather +than asserted. Output is also pasted into the README under "what failure looks +like". +""" + +from __future__ import annotations + +import json + +from answerproof import ReceiptBuilder, SigningKey, verify_receipt +from answerproof.schema import Receipt, Signature + +SOURCES = { + "s1": "The Eiffel Tower is a wrought-iron lattice tower in Paris, France.", + "s2": "It was completed in 1889 and stands 330 metres tall.", + "s3": "The Louvre is the world's most-visited museum, also in Paris.", +} + +ANSWER = ( + "The Eiffel Tower is a wrought-iron lattice tower in Paris, France. " + "It was completed in 1889 and stands 330 metres tall." +) + + +def build_receipt(signing_key: SigningKey) -> Receipt: + builder = ReceiptBuilder(signing_key) + builder.set_query("Tell me about the Eiffel Tower.") + builder.set_answer(ANSWER) + builder.set_principal("user-42", permissions=["kb:paris"], tenant="acme") + builder.set_model("demo-llm", provider="local", params={"temperature": 0.0}) + for sid, content in SOURCES.items(): + builder.add_source(sid, content=content, score=0.9) + return builder.finalize() + + +def _reload(receipt: Receipt) -> dict: + return json.loads(receipt.to_json()) + + +def _from_dict(d: dict) -> Receipt: + return Receipt.from_json(json.dumps(d)) + + +def print_verdict(title: str, verdict) -> None: + print(f"\n=== {title} ===") + print(f"valid: {verdict.valid}") + for check in verdict.checks: + mark = "ok" if check.passed else "FAIL" + detail = f" — {check.detail}" if check.detail else "" + print(f" [{mark}] {check.name}{detail}") + if verdict.skipped: + for s in verdict.skipped: + print(f" [skip] {s}") + + +def main() -> None: + signing_key = SigningKey.generate() + receipt = build_receipt(signing_key) + original_pk = receipt.signature.public_key + + # Baseline: genuine receipt with original source contents. + clean = verify_receipt(receipt, source_contents=SOURCES) + print_verdict("Genuine receipt", clean) + if not clean.valid: + raise SystemExit("baseline receipt failed verification") + + # 1. Source document edited after the fact → fails `sources`. + edited_sources = dict(SOURCES) + edited_sources["s1"] = SOURCES["s1"] + " (quietly rewritten)" + v_sources = verify_receipt(receipt, source_contents=edited_sources) + print_verdict("1. Source document edited after signing", v_sources) + + # 2. Claim citation swapped to a different (unknown) source id → fails `grounding`. + d = _reload(receipt) + if d["payload"]["citations"]: + d["payload"]["citations"][0]["source_id"] = "not-a-real-source" + v_grounding = verify_receipt(_from_dict(d), source_contents=SOURCES) + print_verdict("2. Citation swapped to a different source id", v_grounding) + + # 3. Merkle root edited by hand → fails `merkle` (and usually `signature`). + d = _reload(receipt) + d["payload"]["merkle_root"] = "ff" * 32 + v_merkle = verify_receipt(_from_dict(d), source_contents=SOURCES) + print_verdict("3. Merkle root edited by hand", v_merkle) + + # 4. Payload edited and re-signed with a different key. + # Signature alone passes (attacker's key); pinning the real signer catches it. + attacker = SigningKey.generate() + d = _reload(receipt) + d["payload"]["answer"] = "Tampered but re-signed." + forged = Receipt.from_json(json.dumps(d)) + forged.signature = Signature( + public_key=attacker.verify_key.to_base64(), + signature=attacker.sign(forged.payload.canonical_bytes()), + ) + v_resign = verify_receipt(forged, source_contents=SOURCES) + print_verdict("4a. Re-signed with attacker's key (no pin)", v_resign) + v_pinned = verify_receipt( + forged, source_contents=SOURCES, expected_public_key=original_pk + ) + print_verdict("4b. Same receipt, signer pinned to the original key", v_pinned) + + ok = ( + not v_sources.valid + and any(c.name == "sources" and not c.passed for c in v_sources.checks) + and not v_grounding.valid + and any(c.name == "grounding" and not c.passed for c in v_grounding.checks) + and not v_merkle.valid + and any(c.name == "merkle" and not c.passed for c in v_merkle.checks) + and v_resign.valid + and not v_pinned.valid + and any(c.name == "signer_pin" and not c.passed for c in v_pinned.checks) + ) + if ok: + print("\nDemo OK: each failure mode failed the expected check.") + else: + raise SystemExit("Demo failed: unexpected verification outcome.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_verifier.py b/tests/test_verifier.py index ab44fb8..12c03b1 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -68,3 +68,21 @@ def test_serialized_receipt_roundtrips_and_verifies(receipt, sources): restored = Receipt.from_json(receipt.to_json()) assert verify_receipt(restored, source_contents=sources).valid + + +def test_unknown_source_id_in_contents_fails_sources_check(receipt, sources): + """source_contents with an id the receipt never recorded must fail sources. + + Callers sometimes pass a whole content map; unknown ids are a real boundary + and must surface as content-hash failures, not be silently skipped. + """ + contents = dict(sources) + contents["never-retrieved"] = "This document was never part of the retrieval set." + + verdict = verify_receipt(receipt, source_contents=contents) + + assert not verdict.valid + failed = [c for c in verdict.checks if c.name == "sources" and not c.passed] + assert len(failed) == 1 + assert "never-retrieved" in failed[0].detail + assert "unknown source" in failed[0].detail