diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 1ff46fb..5440744 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -4,12 +4,15 @@ on: push: pull_request: +permissions: + contents: read + jobs: verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 with: version: "0.11.28" - run: uv sync --frozen @@ -18,4 +21,3 @@ jobs: - run: uv run pytest - run: uv build - run: uv run tripwire verify - diff --git a/README.md b/README.md index ff3a411..8d9d886 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,12 @@ Tripwire evaluates deterministic sequence rules over AI-agent tool-call traces. The main finding is a provenance failure, not a new attack signature. Eight of the eleven canonical holdout false negatives received attacker-controlled strings inside transaction records returned by banking tools classified as internal. Tool-result-level provenance is too coarse when a trusted tool returns mixed-trust records; item-level provenance is required to distinguish the injected content without distrusting the entire result. +Tripwire is a research prototype and reproducibility artifact. It is not affiliated with Fortra's Tripwire products. GitHub issues may be used to report reproducibility defects; no support or release cadence is promised. + ## Reproduce the committed result +Prerequisites: Python 3.11–3.13 and [uv](https://docs.astral.sh/uv/getting-started/installation/). + ```console uv sync --frozen uv run tripwire verify @@ -41,7 +45,7 @@ This result classifies successful prompt-injection outcomes in the committed Age - [AgentDojo](https://proceedings.neurips.cc/paper_files/paper/2024/hash/97091a5177d8dc64b1da8bf3e1f6fb54-Abstract-Datasets_and_Benchmarks_Track.html) supplies the agent tasks, attacks, benchmark outcomes and source traces used by the committed corpus. Tripwire adds a sanitised replay corpus and detection-oriented measurement rather than a new attack benchmark. - [Agent Threat Rules (ATR)](https://github.com/Agent-Threat-Rule/agent-threat-rules) standardises portable detections over agent events and content fields. -- Cross-event agent detection is established prior art, not Tripwire's novelty. [AgentSigma](https://github.com/cveye/agentsigma) applies Sigma-compatible runtime detection at the tool-call layer; [AgentShield's sigma-ai](https://github.com/agentshield-ai/sigma-ai) documents temporal correlation for sequential agent events; [AIDR Sigma](https://github.com/netzilo/aidr-sigma) describes a persistent per-session behaviour graph for multi-step patterns; and [RSigma](https://github.com/timescale/rsigma) provides in-process stateful correlation and OTLP ingest. RSigma is also a candidate substrate for any future streaming backend and should be evaluated before Tripwire implements those capabilities itself. +- Cross-event agent detection is established prior art, not Tripwire's novelty. [AgentSigma](https://medium.com/@dasgupta.pratip/agentsigma-v0-1-building-the-detection-framework-i-wrote-about-3f6e14c194c5) applies Sigma-compatible runtime detection at the tool-call layer; [AgentShield's sigma-ai](https://github.com/agentshield-ai/sigma-ai) documents temporal correlation for sequential agent events; [AIDR Sigma](https://github.com/netzilo/aidr-sigma) describes a persistent per-session behaviour graph for multi-step patterns; and [RSigma](https://github.com/timescale/rsigma) provides in-process stateful correlation and OTLP ingest. RSigma is also a candidate substrate for any future streaming backend and should be evaluated before Tripwire implements those capabilities itself. - Tripwire's contribution here is the measurement: a sanitised labelled AgentDojo corpus, reproducible metrics, a frozen holdout and the provenance-granularity finding that resulted from error analysis. - [Out-of-band agent defenses](https://arxiv.org/abs/2606.26479) use deterministic policies, capabilities or information-flow labels outside the model. Tripwire's finding identifies a boundary condition for designs that collapse provenance to tool identity or whole tool results: a trusted tool can return attacker-controlled fields. diff --git a/pyproject.toml b/pyproject.toml index 38d2c51..2ac7292 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ tripwire = "tripwire.cli:main" dev = [ "build==1.2.2.post1", "mypy==1.15.0", - "pytest==8.3.5", + "pytest==9.0.3", "ruff==0.9.10", "types-PyYAML==6.0.12.20250516", ] diff --git a/src/tripwire/reporting.py b/src/tripwire/reporting.py index 969f582..cbf228e 100644 --- a/src/tripwire/reporting.py +++ b/src/tripwire/reporting.py @@ -20,8 +20,12 @@ def render_readme(result: dict[str, Any]) -> str: The main finding is a provenance failure, not a new attack signature. Eight of the eleven canonical holdout false negatives received attacker-controlled strings inside transaction records returned by banking tools classified as internal. Tool-result-level provenance is too coarse when a trusted tool returns mixed-trust records; item-level provenance is required to distinguish the injected content without distrusting the entire result. +Tripwire is a research prototype and reproducibility artifact. It is not affiliated with Fortra's Tripwire products. GitHub issues may be used to report reproducibility defects; no support or release cadence is promised. + ## Reproduce the committed result +Prerequisites: Python 3.11–3.13 and [uv](https://docs.astral.sh/uv/getting-started/installation/). + ```console uv sync --frozen uv run tripwire verify @@ -57,7 +61,7 @@ def render_readme(result: dict[str, Any]) -> str: - [AgentDojo](https://proceedings.neurips.cc/paper_files/paper/2024/hash/97091a5177d8dc64b1da8bf3e1f6fb54-Abstract-Datasets_and_Benchmarks_Track.html) supplies the agent tasks, attacks, benchmark outcomes and source traces used by the committed corpus. Tripwire adds a sanitised replay corpus and detection-oriented measurement rather than a new attack benchmark. - [Agent Threat Rules (ATR)](https://github.com/Agent-Threat-Rule/agent-threat-rules) standardises portable detections over agent events and content fields. -- Cross-event agent detection is established prior art, not Tripwire's novelty. [AgentSigma](https://github.com/cveye/agentsigma) applies Sigma-compatible runtime detection at the tool-call layer; [AgentShield's sigma-ai](https://github.com/agentshield-ai/sigma-ai) documents temporal correlation for sequential agent events; [AIDR Sigma](https://github.com/netzilo/aidr-sigma) describes a persistent per-session behaviour graph for multi-step patterns; and [RSigma](https://github.com/timescale/rsigma) provides in-process stateful correlation and OTLP ingest. RSigma is also a candidate substrate for any future streaming backend and should be evaluated before Tripwire implements those capabilities itself. +- Cross-event agent detection is established prior art, not Tripwire's novelty. [AgentSigma](https://medium.com/@dasgupta.pratip/agentsigma-v0-1-building-the-detection-framework-i-wrote-about-3f6e14c194c5) applies Sigma-compatible runtime detection at the tool-call layer; [AgentShield's sigma-ai](https://github.com/agentshield-ai/sigma-ai) documents temporal correlation for sequential agent events; [AIDR Sigma](https://github.com/netzilo/aidr-sigma) describes a persistent per-session behaviour graph for multi-step patterns; and [RSigma](https://github.com/timescale/rsigma) provides in-process stateful correlation and OTLP ingest. RSigma is also a candidate substrate for any future streaming backend and should be evaluated before Tripwire implements those capabilities itself. - Tripwire's contribution here is the measurement: a sanitised labelled AgentDojo corpus, reproducible metrics, a frozen holdout and the provenance-granularity finding that resulted from error analysis. - [Out-of-band agent defenses](https://arxiv.org/abs/2606.26479) use deterministic policies, capabilities or information-flow labels outside the model. Tripwire's finding identifies a boundary condition for designs that collapse provenance to tool identity or whole tool results: a trusted tool can return attacker-controlled fields. diff --git a/tests/test_m1_acceptance.py b/tests/test_m1_acceptance.py index fe0fc44..c3f3c11 100644 --- a/tests/test_m1_acceptance.py +++ b/tests/test_m1_acceptance.py @@ -380,10 +380,13 @@ def test_public_report_preserves_evaluation_discipline() -> None: assert "reports recall only" in readme assert "## Related work" in readme assert "Cross-event agent detection is established prior art" in readme + assert "medium.com/@dasgupta.pratip/agentsigma-v0-1" in readme + assert "github.com/cveye/agentsigma" not in readme assert "github.com/agentshield-ai/sigma-ai" in readme assert "github.com/netzilo/aidr-sigma" in readme assert "github.com/timescale/rsigma" in readme assert "Tripwire's contribution here is the measurement" in readme + assert "It is not affiliated with Fortra's Tripwire products" in readme assert any("holdout is spent" in limitation for limitation in result["limitations"]) diff --git a/uv.lock b/uv.lock index 411457f..76c755f 100644 --- a/uv.lock +++ b/uv.lock @@ -92,6 +92,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + [[package]] name = "pyproject-hooks" version = "1.2.0" @@ -103,17 +112,18 @@ wheels = [ [[package]] name = "pytest" -version = "8.3.5" +version = "9.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, + { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload-time = "2025-03-02T12:54:54.503Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634, upload-time = "2025-03-02T12:54:52.069Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] [[package]] @@ -200,7 +210,7 @@ requires-dist = [{ name = "pyyaml", specifier = "==6.0.2" }] dev = [ { name = "build", specifier = "==1.2.2.post1" }, { name = "mypy", specifier = "==1.15.0" }, - { name = "pytest", specifier = "==8.3.5" }, + { name = "pytest", specifier = "==9.0.3" }, { name = "ruff", specifier = "==0.9.10" }, { name = "types-pyyaml", specifier = "==6.0.12.20250516" }, ]