Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ pip install "answerproof[api]" # + FastAPI verifier service

## Quickstart

```bash
pip install answerproof
```

```python
from answerproof import ReceiptBuilder, SigningKey, verify_receipt

Expand All @@ -86,6 +90,21 @@ verdict = verify_receipt(
source_contents={"doc-1": "The Eiffel Tower is 330 metres tall."},
)
assert verdict.valid
print("valid:", verdict.valid)
for check in verdict.checks:
mark = "ok" if check.passed else "FAIL"
detail = f" — {check.detail}" if check.detail else ""
print(f" [{mark}] {check.name}{detail}")
```

Expected verification output (ids, timestamps, and digests will differ each run):

```text
valid: True
[ok] signature
[ok] merkle
[ok] grounding — citations and grounding are self-consistent
[ok] sources — 1 source content(s) matched
```

## Receipt anatomy
Expand Down Expand Up @@ -147,6 +166,66 @@ Every check is independent and reported separately:
| `grounding` | citations reference real sources; grounding score is honest |
| `signer_pin`| (optional) signer public key matches an expected key |

### What failure looks like

`examples/tamper.py` builds a valid receipt, then walks each failure mode and
prints the verdict. Run it with:

```bash
python examples/tamper.py
```

Representative output (keys and digests vary per run):

```text
=== Genuine receipt ===
valid: True
[ok] signature
[ok] merkle
[ok] grounding — citations and grounding are self-consistent
[ok] sources — 3 source content(s) matched

=== 1. Source document edited after signing ===
valid: False
[ok] signature
[ok] merkle
[ok] grounding — citations and grounding are self-consistent
[FAIL] sources — content hash mismatch: s1

=== 2. Citation swapped to a different source id ===
valid: False
[FAIL] signature — Ed25519 signature does not match payload
[ok] merkle
[FAIL] grounding — citation references unknown source not-a-real-source
[ok] sources — 3 source content(s) matched

=== 3. Merkle root edited by hand ===
valid: False
[FAIL] signature — Ed25519 signature does not match payload
[FAIL] merkle — recomputed Merkle root does not match signed root
[ok] grounding — citations and grounding are self-consistent
[ok] sources — 3 source content(s) matched

=== 4a. Re-signed with attacker's key (no pin) ===
valid: True
[ok] signature
[ok] merkle
[ok] grounding — citations and grounding are self-consistent
[ok] sources — 3 source content(s) matched

=== 4b. Same receipt, signer pinned to the original key ===
valid: False
[FAIL] signer_pin — receipt public key does not match expected signer
[ok] signature
[ok] merkle
[ok] grounding — citations and grounding are self-consistent
[ok] sources — 3 source content(s) matched
```

Case 4 is the interesting one: re-signing with a different key makes the
signature check pass again. Only pinning the expected public key (`signer_pin`)
catches the substitution — integrity alone is not provenance.

## Merkle inclusion proofs

Prove one source was in the retrieval set without revealing the rest:
Expand Down Expand Up @@ -218,6 +297,7 @@ cd answerproof
pip install -e ".[dev]"
pytest -q # full test suite
python examples/demo_rag.py # produce + verify a receipt, then tamper and re-verify
python examples/tamper.py # walk each failure mode (sources, grounding, merkle, re-sign)
```

## Contributing
Expand Down
127 changes: 127 additions & 0 deletions examples/tamper.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
"""Walk each verification failure mode so newcomers can see what a bad receipt looks like.

Run with no external services:

python examples/tamper.py

Pairs with ``tests/test_tamper.py``: same mutations, printed as a demo rather
than asserted. Output is also pasted into the README under "what failure looks
like".
"""

from __future__ import annotations

import json

from answerproof import ReceiptBuilder, SigningKey, verify_receipt
from answerproof.schema import Receipt, Signature

SOURCES = {
"s1": "The Eiffel Tower is a wrought-iron lattice tower in Paris, France.",
"s2": "It was completed in 1889 and stands 330 metres tall.",
"s3": "The Louvre is the world's most-visited museum, also in Paris.",
}

ANSWER = (
"The Eiffel Tower is a wrought-iron lattice tower in Paris, France. "
"It was completed in 1889 and stands 330 metres tall."
)


def build_receipt(signing_key: SigningKey) -> Receipt:
builder = ReceiptBuilder(signing_key)
builder.set_query("Tell me about the Eiffel Tower.")
builder.set_answer(ANSWER)
builder.set_principal("user-42", permissions=["kb:paris"], tenant="acme")
builder.set_model("demo-llm", provider="local", params={"temperature": 0.0})
for sid, content in SOURCES.items():
builder.add_source(sid, content=content, score=0.9)
return builder.finalize()


def _reload(receipt: Receipt) -> dict:
return json.loads(receipt.to_json())


def _from_dict(d: dict) -> Receipt:
return Receipt.from_json(json.dumps(d))


def print_verdict(title: str, verdict) -> None:
print(f"\n=== {title} ===")
print(f"valid: {verdict.valid}")
for check in verdict.checks:
mark = "ok" if check.passed else "FAIL"
detail = f" — {check.detail}" if check.detail else ""
print(f" [{mark}] {check.name}{detail}")
if verdict.skipped:
for s in verdict.skipped:
print(f" [skip] {s}")


def main() -> None:
signing_key = SigningKey.generate()
receipt = build_receipt(signing_key)
original_pk = receipt.signature.public_key

# Baseline: genuine receipt with original source contents.
clean = verify_receipt(receipt, source_contents=SOURCES)
print_verdict("Genuine receipt", clean)
if not clean.valid:
raise SystemExit("baseline receipt failed verification")

# 1. Source document edited after the fact → fails `sources`.
edited_sources = dict(SOURCES)
edited_sources["s1"] = SOURCES["s1"] + " (quietly rewritten)"
v_sources = verify_receipt(receipt, source_contents=edited_sources)
print_verdict("1. Source document edited after signing", v_sources)

# 2. Claim citation swapped to a different (unknown) source id → fails `grounding`.
d = _reload(receipt)
if d["payload"]["citations"]:
d["payload"]["citations"][0]["source_id"] = "not-a-real-source"
v_grounding = verify_receipt(_from_dict(d), source_contents=SOURCES)
print_verdict("2. Citation swapped to a different source id", v_grounding)

# 3. Merkle root edited by hand → fails `merkle` (and usually `signature`).
d = _reload(receipt)
d["payload"]["merkle_root"] = "ff" * 32
v_merkle = verify_receipt(_from_dict(d), source_contents=SOURCES)
print_verdict("3. Merkle root edited by hand", v_merkle)

# 4. Payload edited and re-signed with a different key.
# Signature alone passes (attacker's key); pinning the real signer catches it.
attacker = SigningKey.generate()
d = _reload(receipt)
d["payload"]["answer"] = "Tampered but re-signed."
forged = Receipt.from_json(json.dumps(d))
forged.signature = Signature(
public_key=attacker.verify_key.to_base64(),
signature=attacker.sign(forged.payload.canonical_bytes()),
)
v_resign = verify_receipt(forged, source_contents=SOURCES)
print_verdict("4a. Re-signed with attacker's key (no pin)", v_resign)
v_pinned = verify_receipt(
forged, source_contents=SOURCES, expected_public_key=original_pk
)
print_verdict("4b. Same receipt, signer pinned to the original key", v_pinned)

ok = (
not v_sources.valid
and any(c.name == "sources" and not c.passed for c in v_sources.checks)
and not v_grounding.valid
and any(c.name == "grounding" and not c.passed for c in v_grounding.checks)
and not v_merkle.valid
and any(c.name == "merkle" and not c.passed for c in v_merkle.checks)
and v_resign.valid
and not v_pinned.valid
and any(c.name == "signer_pin" and not c.passed for c in v_pinned.checks)
)
if ok:
print("\nDemo OK: each failure mode failed the expected check.")
else:
raise SystemExit("Demo failed: unexpected verification outcome.")


if __name__ == "__main__":
main()
18 changes: 18 additions & 0 deletions tests/test_verifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,21 @@ def test_serialized_receipt_roundtrips_and_verifies(receipt, sources):

restored = Receipt.from_json(receipt.to_json())
assert verify_receipt(restored, source_contents=sources).valid


def test_unknown_source_id_in_contents_fails_sources_check(receipt, sources):
"""source_contents with an id the receipt never recorded must fail sources.

Callers sometimes pass a whole content map; unknown ids are a real boundary
and must surface as content-hash failures, not be silently skipped.
"""
contents = dict(sources)
contents["never-retrieved"] = "This document was never part of the retrieval set."

verdict = verify_receipt(receipt, source_contents=contents)

assert not verdict.valid
failed = [c for c in verdict.checks if c.name == "sources" and not c.passed]
assert len(failed) == 1
assert "never-retrieved" in failed[0].detail
assert "unknown source" in failed[0].detail
Loading