diff --git a/.github/actions/plugin-catalog-update/catalog_update.py b/.github/actions/plugin-catalog-update/catalog_update.py index 5ffa2ef..f725090 100644 --- a/.github/actions/plugin-catalog-update/catalog_update.py +++ b/.github/actions/plugin-catalog-update/catalog_update.py @@ -272,6 +272,39 @@ def parse_predicates(spec: str) -> list[tuple[str, str | None]]: return rows +def _summarize_verify(stdout: str) -> str: + """Distil `gh attestation verify --format json` into a compact, readable + evidence block (predicate, signer identity, issuer) for the PR body. Falls + back to the raw text if it is not the expected JSON.""" + try: + recs = json.loads(stdout) + except (json.JSONDecodeError, TypeError): + return stdout.strip() + if not isinstance(recs, list): + return stdout.strip() + + def _d(v: object) -> dict: + return v if isinstance(v, dict) else {} + + blocks = [] + for r in recs: + vr = _d(_d(r).get("verificationResult")) + cert = _d(_d(vr.get("signature")).get("certificate")) + stmt = _d(vr.get("statement")) + predicate = stmt.get("predicateType") + signer = cert.get("buildSignerURI") or cert.get("subjectAlternativeName") + issuer = cert.get("issuer") + if not (predicate or signer or issuer): + continue # non-dict / junk record with nothing usable — skip it + blocks.append( + f"predicate: {predicate or '?'}\n" + f"signer: {signer or '?'}\n" + f"issuer: {issuer or '?'}" + ) + # Empty/unexpected shape -> fall back to raw so the evidence block is never blank. + return "\n\n".join(blocks) if blocks else stdout.strip() + + def verify_subject(subject: str, repo: str, predicates: list[tuple[str, str | None]]) -> list[dict]: """Verify one subject (file path or oci ref) against each required predicate. @@ -281,16 +314,22 @@ def verify_subject(subject: str, repo: str, predicates: list[tuple[str, str | No """ results = [] for predicate, signer in predicates: - cmd = ["attestation", "verify", subject, "--repo", repo, "--predicate-type", predicate] + cmd = ["attestation", "verify", subject, "--repo", repo, + "--predicate-type", predicate, "--format", "json"] if signer: cmd += ["--signer-workflow", signer] proc = gh(*cmd, check=False) + ok = proc.returncode == 0 + # gh attestation verify suppresses its human-readable summary when stdout + # is not a TTY (headless CI), leaving stdout+stderr empty on success — so + # capture --format json and distil it; on failure stderr carries the error. + output = _summarize_verify(proc.stdout) if ok else (proc.stdout + proc.stderr).strip() results.append( { "predicate": predicate, "signer": signer or f"repo:{repo}", - "ok": proc.returncode == 0, - "output": (proc.stdout + proc.stderr).strip(), + "ok": ok, + "output": output, } ) return results @@ -341,14 +380,16 @@ def render_pr_body(plugin_name: str, repo: str, old_ref: str, new_ref: str, # One re-verify command per predicate, carrying --signer-workflow for # seam-signed predicates so a reviewer reproduces exactly what was verified # (--repo alone is insufficient for seam-signed gates; org CLAUDE.md §5). + # --format json mirrors how the engine verifies, so the command produces + # evidence even in a non-TTY context (plain output is suppressed headless). if evidence["checks"]: reverify = "\n".join( - f"gh attestation verify {subject} --repo {repo} --predicate-type {c['predicate']}" + f"gh attestation verify {subject} --repo {repo} --predicate-type {c['predicate']} --format json" + ("" if c["signer"].startswith("repo:") else f" \\\n --signer-workflow {c['signer']}") for c in evidence["checks"] ) else: - reverify = f"gh attestation verify {subject} --repo {repo} --predicate-type " + reverify = f"gh attestation verify {subject} --repo {repo} --predicate-type --format json" return f"""\ Automated, **verify-first** re-pin of an external plugin to its latest attested release. diff --git a/.github/actions/plugin-catalog-update/test_catalog_update.py b/.github/actions/plugin-catalog-update/test_catalog_update.py index 54810ef..81424d5 100644 --- a/.github/actions/plugin-catalog-update/test_catalog_update.py +++ b/.github/actions/plugin-catalog-update/test_catalog_update.py @@ -290,5 +290,39 @@ def test_missing_plugin_raises(self): cu.repin_text(self.MP, "nope", "a" * 40, "b" * 40, "v0.1.1") + +class SummarizeVerify(unittest.TestCase): + JSON = ( + '[{"verificationResult": {"statement": {"predicateType": "https://slsa.dev/provenance/v1"},' + ' "signature": {"certificate": {"buildSignerURI": "https://github.com/o/r/.github/workflows/a.yml@refs/tags/v1",' + ' "issuer": "https://token.actions.githubusercontent.com"}}}}]' + ) + + def test_distills_signer_and_predicate(self): + out = cu._summarize_verify(self.JSON) + self.assertIn("https://slsa.dev/provenance/v1", out) + self.assertIn("a.yml@refs/tags/v1", out) + self.assertIn("token.actions.githubusercontent.com", out) + + def test_non_json_falls_back_to_raw(self): + self.assertEqual(cu._summarize_verify("plain text"), "plain text") + + def test_empty_list_falls_back_to_raw(self): + self.assertEqual(cu._summarize_verify("[]"), "[]") + + def test_unexpected_shape_falls_back(self): + self.assertEqual(cu._summarize_verify('{"a": 1}'), '{"a": 1}') + + def test_skips_non_dict_records(self): + # a list with a non-dict record must not crash; falls back when no blocks + self.assertEqual(cu._summarize_verify('[1, 2]'), '[1, 2]') + + def test_malformed_nested_falls_back(self): + # verificationResult (or signature/statement) being a non-dict must not crash + for raw in ('[{"verificationResult": "x"}]', + '[{"verificationResult": {"signature": "x", "statement": 3}}]'): + self.assertEqual(cu._summarize_verify(raw), raw) + + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/engine-tests.yml b/.github/workflows/engine-tests.yml new file mode 100644 index 0000000..9f4645c --- /dev/null +++ b/.github/workflows/engine-tests.yml @@ -0,0 +1,39 @@ +# Repo-CI gate (not a reusable): lint + unit-test the catalog-updater engine on +# every change to it, so a regression can't merge unseen. The engine +# (.github/actions/plugin-catalog-update) is plain-stdlib Python that shells to +# `gh`; its tests are pure (no network) and run in milliseconds. Pin ruff to a +# verified version so a new lint rule can't break CI without a deliberate bump. +name: engine-tests + +on: + pull_request: + paths: + - '.github/actions/plugin-catalog-update/**' + push: + branches: [main] + paths: + - '.github/actions/plugin-catalog-update/**' + +permissions: + contents: read + +jobs: + catalog-update-engine: + name: catalog-update-engine + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Lint + unit-test the engine + working-directory: .github/actions/plugin-catalog-update + env: + RUFF_VERSION: 0.14.14 + run: | + set -euo pipefail + python3 -m venv "${RUNNER_TEMP}/ruff-venv" + "${RUNNER_TEMP}/ruff-venv/bin/pip" install --quiet "ruff==${RUFF_VERSION}" + "${RUNNER_TEMP}/ruff-venv/bin/ruff" check catalog_update.py test_catalog_update.py + python3 -m unittest test_catalog_update -v