diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab99e74..3dd45c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - run: ruff format --check src tests tools - run: ruff check src tests tools - run: mypy src - - run: pytest --cov-report=xml + - run: pytest --cov=agentguard --cov-report=xml --cov-report=term-missing --cov-fail-under=85 # Detection quality is a reviewable number, not a claim. Printed on every run so a # precision or recall change shows up in the diff of two CI logs. - run: python -m tools.bench diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d3ea7fe..a361228 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,43 @@ permissions: contents: read jobs: + # Nothing left this machine before this job existed. `git tag` on a broken commit + # published a broken package, and a PyPI version can be yanked but never replaced - so + # the recovery from a bad release is a new version number and a permanent gap. The + # release proves itself here or not at all. + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: python -m pip install -e '.[dev]' + + # The tag is the only thing a human types in this pipeline, and the version is + # written in exactly one place. If they disagree, PyPI would receive a package + # declaring a version nobody tagged, permanently. + - name: Tag must match the declared version + run: | + tag="${GITHUB_REF_NAME#v}" + declared="$(python -c 'import tomllib,pathlib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')" + installed="$(python -c 'import importlib.metadata as m; print(m.version("agentguard-sast"))')" + echo "tag=$tag pyproject=$declared installed=$installed" + if [ "$tag" != "$declared" ] || [ "$tag" != "$installed" ]; then + echo "::error::tag '$tag' disagrees with pyproject ('$declared') / installed ('$installed')" + exit 1 + fi + + - run: ruff format --check src tests tools + - run: ruff check src tests tools + - run: mypy src + - run: pytest --cov=agentguard --cov-fail-under=85 + - run: python -m tools.bench + - run: python -m tools.bench --field-only + - run: python -m tools.measure_linearity --check + build: + needs: verify runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -17,6 +53,9 @@ jobs: python-version: "3.12" - run: python -m pip install build - run: python -m build + # Paired with download-artifact@v8 below. The majors are not parallel: v8 is the + # documented counterpart to upload-artifact@v7, per download-artifact's release + # notes ("v8 supports downloading artifacts uploaded with actions/upload-artifact@v7"). - uses: actions/upload-artifact@v7 with: name: distributions @@ -31,7 +70,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: distributions path: dist/ @@ -43,7 +82,7 @@ jobs: permissions: contents: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: distributions path: dist/ diff --git a/.gitignore b/.gitignore index 62afaae..003ac9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ __pycache__/ *.py[cod] *.egg-info/ +# Root-anchored on purpose. A bare `.env` would also ignore the corpus fixtures at +# tests/corpus/**, which must stay tracked - they are the test data for .env scanning. +/.env +/.env.* +!/.env.example + .venv/ .venv*/ venv/ diff --git a/Makefile b/Makefile index 8e9ec54..6bb8b95 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ type: mypy src test: - pytest + pytest --cov=agentguard --cov-report=term-missing --cov-fail-under=85 bench: python -m tools.bench diff --git a/README.md b/README.md index fb4ff0c..bed9e2c 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,16 @@ Measured against a labelled corpus in this repository. Reproduce it with `make b | **all** | 14 | 1 | 0 | 93.3% | 100.0% | ``` -**93.3% precision and 100% recall over 34 labelled files** — 13 true positives and 21 true -negatives, each carrying a written reason for its label in `tests/corpus/manifest.yml`. -That is the only accuracy figure this project publishes, because it is the only one it can -reproduce. +**93.3% precision and 100% recall over 34 labelled files** — 14 that must produce a +finding and 20 that must not, each carrying a written reason for its label in +`tests/corpus/manifest.yml`. That is the only accuracy figure this project publishes, +because it is the only one it can reproduce. -13 of the 21 true negatives reproduce a false positive observed on a real project; the other +The table counts *findings*, not files: 14 true positives from the 14 positive files, and +the single false positive is one finding on one negative file. They are one-to-one here +because no corpus case expects more than one rule to fire. + +12 of the 20 negative cases reproduce a false positive observed on a real project; the other 8 were composed to cover awkward cases — a credential in a docstring, `eval` on a literal, `subprocess` with a fixed argument vector, a `.env` full of shell interpolation. Every case declares which it is, and `make bench` scores the field-derived subset separately: diff --git a/WORKLOG.md b/WORKLOG.md index 996f815..85a8707 100644 --- a/WORKLOG.md +++ b/WORKLOG.md @@ -1281,3 +1281,326 @@ Decisions taken alone: 1. **Field-only reports "behave as labelled" rather than leading with precision**, because precision over a subset selected from observed failures is skewed by construction. 2. **Unit D closed without a commit**, the framing already being present and verified. + +--- + +## Unit 16 — R1: artifact actions aligned to the documented pairing — 2026-07-29 + +Status: complete +Changed: `.github/workflows/release.yml` + +Verified from the actions' own release notes, not from the assumption that equal majors +interoperate — and the majors are **not** parallel. `upload-artifact` is at v7, +`download-artifact` at v8. `download-artifact`'s README states v8 "supports downloading +artifacts uploaded with `actions/upload-artifact@v7`", so v8 is the counterpart to v7 and +both download steps are now v8. + +### Correction to the finding's severity + +The mismatch was real. The stated failure mode — publish failing after the tag is already +pushed — **would not have happened today**, and the reasoning matters for judging what else +to trust: + +- `upload-artifact@v7`'s direct (unzipped) upload is **opt-in** via `archive: false`, and + the release job does not set it. +- Direct upload supports a single file only and fails on a glob resolving to several. + `path: dist/` is a directory containing a wheel and an sdist, so the feature cannot apply. +- So v7 produced a zipped artifact, which `download-artifact@v4` unzips correctly. + +The real costs of leaving it were different and less dramatic: `download-artifact@v4` runs +on **Node 20, which GitHub is deprecating** on its runners (already emitting warnings in +this repository's dependency-review logs), while the build side had moved to Node 24; and +v8 defaults `digest-mismatch` to `error` rather than a warning, which is the behaviour a +release pipeline wants — a corrupted download should stop a publish, not log about it. + +Recording this because the queue asked for verification rather than assumption, and +verification changed the answer: right fix, different reason, lower urgency than stated. + +Bench delta: n/a — workflow only. +Decisions taken alone: none. +Next: R2 + R3, which share the `verify` job. + +--- + +## Unit 17 — R2 + R3: the release proves itself, and the version has one home — 2026-07-29 + +Status: complete +Changed: `.github/workflows/release.yml`, `pyproject.toml`, `src/agentguard/__init__.py`, +`tests/test_cli.py` + +### The version lived in two places, not one + +The finding named `pyproject.toml:7`. It was also hardcoded at +`src/agentguard/__init__.py:7`, and **that** is the one that feeds `--version`, the JSON +report's `tool.version`, and the SARIF driver version. Bumping only `pyproject.toml` would +have published a package declaring `0.2.0` while every report it emitted claimed `0.1.0` — +including SARIF uploaded into GitHub code scanning. Worse than the finding described, and +only visible by grepping for the literal rather than reading the named line. + +### hatch-vcs versus a comparison check — chose the check, plus single-sourcing + +`hatch-vcs` was rejected, for two reasons. It adds a build-time dependency to a project +whose pitch includes a three-package runtime tree, and it makes the version depend on git +history being present at build time, which complicates building from an sdist. Neither is +fatal; both are cost for a problem that a five-line check solves. + +But a comparison check alone would have left the duplication, and the duplication is the +actual defect. So `__init__.py` now reads `importlib.metadata.version("agentguard-sast")`, +making `pyproject.toml` the only place a version number is written, with a +`PackageNotFoundError` fallback for a source tree with no install. `tests/test_cli.py` +asserts the CLI output against the metadata rather than a literal — a hardcoded assertion +there would be a third site to edit in lockstep, which is the drift it should catch, not +cause. + +Version is now `0.2.0`. + +### The verify job + +`publish` reached PyPI without anything having run. There is now a `verify` job that +`build` depends on, and it runs the full suite, both bench scopes, the linearity gate, lint, +mypy, and the tag/version consistency check. + +Chain: `verify → build → publish → github-release`. + +Tag check exercised against agreeing and disagreeing inputs, because a gate that cannot +fail is the failure mode this whole queue is about: + +``` +GITHUB_REF_NAME=v0.2.0 -> tag=0.2.0 pyproject=0.2.0 installed=0.2.0 pass +GITHUB_REF_NAME=v0.2.1 -> tag=0.2.1 pyproject=0.2.0 installed=0.2.0 FAIL (blocks release) +GITHUB_REF_NAME=v1.0.0 -> tag=1.0.0 pyproject=0.2.0 installed=0.2.0 FAIL (blocks release) +``` +It compares the tag against both `pyproject.toml` and the installed metadata, so the +single-sourcing cannot silently come undone either. + +Trusted publishing untouched: `id-token: write`, `environment: pypi`, and `--verify-tag` +are as they were. + +Verified: lint, mypy, 197 passed + 1 xfail, bench 33 of 34. +Bench delta: none. +Decisions taken alone: +1. **Comparison check over `hatch-vcs`**, reasoning above. +2. **`__version__` derived from installed metadata**, which the finding did not ask for but + which is the underlying defect — a check that only compared the tag to `pyproject.toml` + would have passed while reports still said `0.1.0`. +Next: R4, the sdist. + +--- + +## Unit 18 — R4 to R7: packaging, ignores, coverage flags, py.typed — 2026-07-29 + +Status: complete +Changed: `pyproject.toml`, `tools/bench.py`, `.gitignore`, `Makefile`, +`.github/workflows/ci.yml`, `.github/workflows/release.yml`, `src/agentguard/py.typed` (new) + +### R4 — sdist shipped 36 corpus entries; now zero + +Confirmed by building and listing rather than reading config: + +``` +before sdist 88 entries, 36 corpus wheel 20 entries, 0 corpus +after sdist 39 entries, 0 corpus wheel 20 entries, 0 corpus +``` +Checked for corpus paths, `.env` files, `requirements.txt`, and anything under `tests/`. +Both artifacts: 0 suspect. + +**Chose full `/tests` exclusion over corpus-only, and the reason is not cosmetic.** +Excluding only `/tests/corpus` leaves `tests/test_corpus.py` in the sdist, where it cannot +pass — the manifest it loads is gone. A distribution packager running the bundled suite +would see failures that are not defects, which is worse than shipping no suite at all. The +suite is a `git clone` away, and the new `verify` job runs it from the repository, which is +the only place it is meaningful. + +`tools/bench.py` now detects a missing corpus and says why, instead of emitting 34 +"labelled but missing from disk" errors to anyone who pip-installs the sdist and tries it. + +The pattern the finding names is worth restating, because this is its third instance: +**any file whose shape implies a role — `.env`, `requirements.txt`, `package.json`, +lockfiles, workflow YAML — will be interpreted by something, whatever directory it is in.** +Previous two: GitHub's dependency graph indexing the corpus as a project manifest, and +dependency review failing on an advisory against a fixture's fictional dependency. + +### R5 — root `.env` now ignored, corpus untouched + +Root-anchored, verified with `git check-ignore -v` rather than by reading the pattern: + +``` +IGNORED .env <- .gitignore:6:/.env +IGNORED .env.local <- .gitignore:7:/.env.* +tracked tests/corpus/true_positives/.env +tracked tests/corpus/true_negatives/.env.local +tracked tests/corpus/true_negatives/.env.example +``` + +### R6 — coverage flags out of `addopts` + +`addopts` is now `-ra --strict-markers`. The gate moved to `make test`, `ci.yml`, and the +release `verify` job. Verified in all three directions: + +``` +pytest tests/test_rules.py -> exit 0 (21 passed; was a coverage failure) +pytest --cov-fail-under=85 -> exit 0 (93.44%) +pytest --cov-fail-under=99 -> exit 1 ("Required test coverage of 99% not reached") +``` +The last one matters: moving a gate is only safe if you can show it still bites. + +### R7 — `py.typed` added, and confirmed packaged + +Verified absent first, then added, then confirmed in both artifacts: +`agentguard/py.typed` in the wheel, `src/agentguard/py.typed` in the sdist. Hatchling +picks it up from the package directory with no config change. + +Verified: lint, mypy, 197 passed + 1 xfailed. +Bench delta: none — packaging and configuration only. +Decisions taken alone: +1. **Full `/tests` exclusion**, reasoning above — a half-excluded suite is worse than none. +2. **`tools/` kept in the sdist.** `measure_linearity` works standalone from it; `bench` + now fails with an explanation rather than a wall of missing-file errors. +Next: pre-tag verification — `twine check`, the licence classifier, and a TestPyPI dry run. + +--- + +## Unit 19 — licence metadata, corpus layout, README prose — 2026-07-29 + +Status: complete +Changed: `pyproject.toml`, `README.md`, `tests/corpus/` (one file moved), +`tests/corpus/manifest.yml`, `tests/test_corpus.py` + +### Licence classifier — `twine check` did not warn, but the metadata was wrong anyway + +The dry run was clean on both artifacts, with no deprecation warning about +`License :: OSI Approved :: Apache Software License`. So the stated trigger did not fire. + +What it did surface is worse than a warning, and only visible by reading the built metadata: +`license = {file = "LICENSE"}` under Metadata-Version 2.4 put the **entire Apache licence +text** into the `License:` field. That renders as a wall of text on the PyPI project page, +which cannot be edited without another release. + +Switched to the PEP 639 form — `license = "Apache-2.0"` plus `license-files = ["LICENSE"]` +— and dropped the classifier, which PEP 639 makes redundant: + +``` +before: License: + Classifier: License :: OSI Approved :: ... +after: License-Expression: Apache-2.0 + License-File: LICENSE +``` +`twine check` passes both artifacts after the change. Done now precisely because this is +page metadata that a release freezes. + +### The 14-versus-13 discrepancy was a layout inconsistency, not two findings in one file + +The hypothesis in the finding was that one file yields two findings. It does not — **no +corpus case expects more than one rule.** The real cause: `publishable_keys.py` sat in +`true_negatives/` while carrying `expect: [AG001]`, because its assertion is about severity +(capped at Low) rather than silence. So 13 positive files produced 14 true positives. + +Fixed at the source rather than explained in prose: the file moved to `true_positives/`, +where its expectation matches its directory. The corpus is now self-consistent — +**14 positives that must fire, 20 negatives that must not, 14 TP findings from 14 files, +one-to-one.** `test_directory_matches_expectation` keeps it that way. + +README prose now separates the two explicitly: "34 labelled files — 14 that must produce a +finding and 20 that must not… The table counts *findings*, not files." + +Counts after the move: field-derived negatives 12 (was 13; `publishable_keys.py` is +field-derived but is no longer a negative), written negatives 8. Field-only scope is still +13 cases — 12 negatives plus that one positive — hence "12 of 13 behave as labelled". + +### Artifacts re-verified after every change + +``` +twine check → PASSED (wheel + sdist) +sdist corpus/tests entries → 0 +wheel corpus entries → 0 +README table vs make bench → MATCH +pytest → 198 passed, 1 xfailed +linearity gate → exit 0 +``` + +Bench delta: none. 93.3% / 100%, 33 of 34, 12 of 13 field-only — unchanged by the move, +which was a relabelling of where a case lives, not of what it asserts. +Decisions taken alone: +1. **PEP 639 licence form**, because the embedded-text rendering is a release-frozen defect + even though nothing warned about it. +2. **Moved the file rather than documenting the discrepancy.** Prose explaining why a number + looks wrong is a worse fix than the number not looking wrong. + +--- + +## Unit 20 — what could not be verified, and the package-name question — 2026-07-29 + +Status: reported, no code change. + +### TestPyPI dry run — NOT DONE, no credential + +The queue asked for a publish to TestPyPI, a `pip install` from it, and a check that +`--version` reports `0.2.0`. **I could not do the upload.** There is no `~/.pypirc`, no +`TWINE_*` or `PYPI_*` environment variable, and TestPyPI needs either a token or a +configured trusted publisher. Saying so rather than implying it happened. + +What I did instead exercises everything except the upload transport — installing each built +artifact into a clean virtualenv: + +``` +from the SDIST agentguard --version -> AgentGuard 0.2.0 + scan tool.version -> 0.2.0 + scan findings -> AG001 Critical, AG002 Critical (incl. a .env) + coverage key present -> True + runtime deps -> pyyaml, rich, typer (+ transitives only) +from the WHEEL agentguard --version -> AgentGuard 0.2.0 + scan exit -> 0 + py.typed -> present in site-packages +``` + +That covers R3 (version consistency end to end), R4 (an sdist without the corpus still +builds, installs, and scans), and R7. It does **not** cover: PyPI's own metadata +validation on receipt, the trusted-publishing OIDC exchange, or how the project page +actually renders. Those need a real upload, and the first tag is the first time any of them +is exercised. + +**Recommendation: run the TestPyPI leg yourself before tagging.** It is the only remaining +unexercised path, and it is the one where a mistake is permanent. A TestPyPI trusted +publisher for this repository plus a temporary workflow_dispatch on `release.yml` pointed at +`https://test.pypi.org/legacy/` would do it. + +### Package name — both names are unclaimed + +Checked rather than assumed: + +``` +pypi.org/pypi/agentguard-sast/json -> HTTP 404 +pypi.org/pypi/agentguard/json -> HTTP 404 +test.pypi.org/pypi/agentguard-sast -> HTTP 404 +``` + +`agentguard` is **free right now**. That changes the shape of the problem: this is not +"someone else holds the good name", it is "the name your own documentation types at a shell +prompt is unclaimed, on a security tool". + +**Recommendation, for you to action — I have not registered anything.** + +1. **Claim `agentguard` on PyPI as a placeholder before tagging**, and keep publishing under + `agentguard-sast`. A placeholder is a real 0.0.0 release with a README saying "this name + is reserved; install `agentguard-sast`". It costs one upload and closes the typosquat + permanently. PyPI does not reserve names without a release, so intent is not enough. +2. Or **publish under `agentguard` and retire `agentguard-sast`** before either has users. + Cleanest end state — the install name matches the command — and this is the last moment + it is free, since nothing is published. Cost: the repository, docs, and the + trusted-publishing configuration all reference `agentguard-sast`, and + `pyproject.toml`'s `name` feeds the `importlib.metadata` lookup added in unit 17. +3. Or **do nothing** and accept that `pip install agentguard` may one day install something + this project did not write. + +I would take (1). It is reversible, cheap, and does not touch a working release pipeline on +the eve of a first tag — whereas (2) is the better end state bought at the cost of changing +the distribution name in the same change that first exercises the publish path. If you want +(2), it should be its own PR, before the tag, not folded into this one. + +Either way this is a decision about a namespace you own, so it stays with you. + +### Not done, deliberately + +- No tag. Version is `0.2.0` in `pyproject.toml`; nothing is tagged. +- Trusted publishing untouched. +- The `[0.1.0] — NEVER PUBLISHED` annotation untouched. +- PRs #12, #13, #14, #15 untouched. #14 adds a pattern to the unbounded AG001 and will be + gated by the linearity job on its own PR now that `--check` runs in CI. diff --git a/pyproject.toml b/pyproject.toml index e544b44..b00aca3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,18 +4,18 @@ build-backend = "hatchling.build" [project] name = "agentguard-sast" -version = "0.1.0" +version = "0.2.0" description = "Static security scanner for AI agent applications" readme = "README.md" requires-python = ">=3.10" -license = {file = "LICENSE"} +license = "Apache-2.0" +license-files = ["LICENSE"] authors = [{name = "AgentGuard contributors"}] keywords = ["ai", "agents", "security", "langchain", "mcp", "sast", "cli"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -52,10 +52,25 @@ Changelog = "https://github.com/amic25/agentguard/blob/main/CHANGELOG.md" packages = ["src/agentguard"] [tool.hatch.build.targets.sdist] -include = ["/src", "/tests", "/tools", "/docs", "/README.md", "/LICENSE", "/pyproject.toml", "/CHANGELOG.md"] +# `/tests` is deliberately absent. The corpus holds credential-shaped strings, `.env` +# files, two `requirements.txt` files, and fixtures using eval/exec/shell=True by design. +# Shipping it means any organisation running a secret scanner over its package cache, or +# mirroring sdists, gets hits on AgentGuard itself - and a security tool that trips +# security scanners is quarantined at exactly the organisations worth adopting it. This is +# the third external consumer to read the corpus as a real artifact, after GitHub's +# dependency graph and dependency review. +# +# Excluding only `/tests/corpus` was rejected: it leaves a test suite in the sdist that +# cannot pass, since `tests/test_corpus.py` requires the corpus. A packager running it +# would see failures that are not defects. All-or-nothing is the honest shape; the suite is +# a `git clone` away, and `verify` in the release workflow runs it from the repository. +include = ["/src", "/tools", "/docs", "/README.md", "/LICENSE", "/pyproject.toml", "/CHANGELOG.md"] [tool.pytest.ini_options] -addopts = "-ra --strict-markers --cov=agentguard --cov-report=term-missing --cov-fail-under=85" +# Coverage flags deliberately absent. With them here, `pytest tests/test_rules.py` fails +# on coverage even though every test passed - which is the first thing a contributor does. +# The gate lives in `make test` and in CI, where it is measuring the whole suite. +addopts = "-ra --strict-markers" testpaths = ["tests"] norecursedirs = ["tests/corpus"] pythonpath = ["."] diff --git a/src/agentguard/__init__.py b/src/agentguard/__init__.py index 82cf284..1f47e98 100644 --- a/src/agentguard/__init__.py +++ b/src/agentguard/__init__.py @@ -1,7 +1,18 @@ """AgentGuard public package API.""" +from importlib.metadata import PackageNotFoundError, version + from agentguard.models import Finding, ScanResult, Severity from agentguard.scanner import Scanner __all__ = ["Finding", "ScanResult", "Scanner", "Severity"] -__version__ = "0.1.0" + +try: + #: Read from installed metadata so `pyproject.toml` is the only place a version number + #: is written. It used to be hardcoded here as well, and this value - not the one in + #: `pyproject.toml` - is what `--version`, the JSON report's `tool.version`, and the + #: SARIF driver version all report. Bumping one and not the other would have published + #: a package declaring one version while every report it emitted claimed another. + __version__ = version("agentguard-sast") +except PackageNotFoundError: # pragma: no cover - source tree with no install + __version__ = "0.0.0+unknown" diff --git a/src/agentguard/py.typed b/src/agentguard/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/corpus/manifest.yml b/tests/corpus/manifest.yml index 3e7c603..dbdc749 100644 --- a/tests/corpus/manifest.yml +++ b/tests/corpus/manifest.yml @@ -105,6 +105,15 @@ true_positives: on its own, where no `tests/` path component is visible. The earlier name made this entry claim to pin the downgrade while never reaching it. + publishable_keys.py: + origin: field + expect: [AG001] + why: >- + Publishable keys must still be reported - the publishable and secret halves are easy + to confuse - but capped at Low, so this is a true positive whose *severity* is the + assertion. Pinned by tests/test_rule_context.py::test_publishable_key_is_capped_at_low. + Measured as Critical false positives in browser-use and langgraph. + true_negatives: .env.local: origin: written @@ -265,11 +274,3 @@ true_negatives: Deleting a temp file the code itself created is not a consequential side effect on user data, and the path is not agent-controlled. Measured in crewAI. - publishable_keys.py: - origin: field - expect: [AG001] - why: >- - Publishable keys must still be reported - the publishable and secret halves are easy - to confuse - but capped at Low, so this is a true positive whose *severity* is the - assertion. Pinned by tests/test_rule_context.py::test_publishable_key_is_capped_at_low. - Measured as Critical false positives in browser-use and langgraph. diff --git a/tests/corpus/true_negatives/publishable_keys.py b/tests/corpus/true_positives/publishable_keys.py similarity index 100% rename from tests/corpus/true_negatives/publishable_keys.py rename to tests/corpus/true_positives/publishable_keys.py diff --git a/tests/test_cli.py b/tests/test_cli.py index 75e27e4..5732784 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -52,7 +52,14 @@ def test_invalid_format(project: Path) -> None: assert result.exit_code == 2 -def test_version() -> None: +def test_version_matches_package_metadata() -> None: + """Asserted against the metadata rather than a literal. + + A hardcoded version here would have to be edited in lockstep with `pyproject.toml` + and `__init__.py`, which is the drift this test should be catching, not causing. + """ + from importlib.metadata import version + result = runner.invoke(app, ["--version"]) assert result.exit_code == 0 - assert "0.1.0" in result.stdout + assert version("agentguard-sast") in result.stdout diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 695c781..0e40a63 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -80,3 +80,20 @@ def test_field_only_scoring_is_a_strict_subset() -> None: labels, origins = load_manifest() field_only = {p: e for p, e in labels.items() if origins[p] == "field"} assert 0 < len(field_only) < len(labels) + + +def test_directory_matches_expectation() -> None: + """`true_positives/` expects findings; `true_negatives/` expects none. + + One file used to sit in `true_negatives/` while expecting AG001 — its assertion was + about severity, not silence. That made the corpus report 14 true positives across 13 + positive files, and the README prose then had to explain a number that only looked + wrong because the layout was inconsistent. The directory is the claim; keep it true. + """ + import yaml + + raw = yaml.safe_load((CORPUS / "manifest.yml").read_text(encoding="utf-8")) + misfiled = [ + f"true_positives/{name}" for name, entry in raw["true_positives"].items() if not entry["expect"] + ] + [f"true_negatives/{name}" for name, entry in raw["true_negatives"].items() if entry["expect"]] + assert not misfiled, f"expectation contradicts the directory: {misfiled}" diff --git a/tools/bench.py b/tools/bench.py index eb6a0e7..f78adf8 100644 --- a/tools/bench.py +++ b/tools/bench.py @@ -58,6 +58,17 @@ def recall(self) -> float | None: def load_manifest() -> tuple[dict[Path, set[str]], dict[Path, str]]: """Return (expectations, origins). Both keyed by path.""" + if not (CORPUS / "manifest.yml").exists(): + # The published sdist omits `/tests` on purpose - the corpus trips secret scanners. + # Say so, rather than emitting 34 "labelled but missing from disk" errors. + print( + f"corpus not found at {CORPUS}.\n" + "The published sdist excludes tests/ deliberately: the corpus holds " + "credential-shaped fixtures that trip secret scanners.\n" + "Clone the repository to run the benchmark.", + file=sys.stderr, + ) + raise SystemExit(2) raw = yaml.safe_load((CORPUS / "manifest.yml").read_text(encoding="utf-8")) labels: dict[Path, set[str]] = {} origins: dict[Path, str] = {}