Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions .github/workflows/agent-completion-enforcement.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:
if test -s trusted-report.json; then
python3 scripts/ci/agent_completion_enforcement.py trusted-report.json .github/agent-lock/trusted-publishers.json "$head" "$PR" > enforcement-verdict.json
else
printf '%s\n' '{"conclusion":"failure","reason":"missing_trusted_publication","details":{}}' > enforcement-verdict.json
python3 scripts/ci/agent_completion_enforcement.py --missing-publication .github/agent-lock/trusted-publishers.json > enforcement-verdict.json
fi
- name: Publish the required head-bound Check run
if: always()
Expand All @@ -71,8 +71,15 @@ jobs:
} catch (error) {
core.warning(error.message);
}
const conclusion = verdict.conclusion === 'success'
? 'success'
// Verifier currently emits success/failure/neutral. The remaining
// values are reserved for future use, but accepted here to avoid
// hard-failing if verifier capabilities are extended.
const allowedConclusions = new Set([
'success', 'failure', 'neutral', 'cancelled', 'timed_out',
'action_required'
]);
const conclusion = allowedConclusions.has(verdict.conclusion)
? verdict.conclusion
: 'failure';
const summary = JSON.stringify(verdict);
await github.rest.checks.create({
Expand All @@ -89,6 +96,6 @@ jobs:
summary: summary.slice(0, 60000)
}
});
if (conclusion !== 'success') {
if (conclusion === 'failure') {
core.setFailed(verdict.reason || 'trusted evidence blocked');
}
56 changes: 49 additions & 7 deletions scripts/ci/agent_completion_enforcement.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,48 @@ def verdict(reason: str, **details: Any) -> dict[str, Any]:
return {"conclusion": "failure", "reason": reason, "details": details}


def _is_non_empty_string_list(value: Any) -> bool:
return isinstance(value, list) and all(
isinstance(item, str) and len(item) > 0 for item in value
)


def _extract_trust_policy_lists(policy: Any) -> tuple[list[str], list[str], list[str]] | None:
if not isinstance(policy, dict):
return None
apps = policy.get("trusted_check_app_slugs")
labels = policy.get("trusted_label_actors")
exemptions = policy.get("trusted_human_exemption_actors")
if not all(_is_non_empty_string_list(value) for value in (apps, labels, exemptions)):
return None
return apps, labels, exemptions


def missing_publication(policy: Any) -> dict[str, Any]:
trusted = _extract_trust_policy_lists(policy)
if trusted is None:
return verdict("invalid_trust_policy")
apps, labels, exemptions = trusted
if not apps or not labels or not exemptions:
return {
"conclusion": "neutral",
Comment on lines +44 to +48

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
return verdict("invalid_trust_policy")
apps, labels, exemptions = trusted
if not apps or not labels or not exemptions:
return {
"conclusion": "neutral",
return verdict("trust_policy_unprovisioned")

The missing_publication() unprovisioned branch returns a non-blocking neutral verdict when trust allowlists are empty, turning the documented fail-closed agent-completion gate into a fail-open one that blocks no PRs.

Fix on Vercel

"reason": "missing_publication_policy_unprovisioned",
"details": {},
}
return verdict("missing_trusted_publication")


def verify(payload: Any, policy: Any, head_sha: str, pull_number: int) -> dict[str, Any]:
if not isinstance(payload, dict) or not isinstance(policy, dict):
return verdict("invalid_payload")
if not SHA.fullmatch(head_sha) or type(pull_number) is not int or pull_number < 1:
return verdict("invalid_invocation")
if policy.get("custom_role_policy") != "fail_closed":
return verdict("invalid_custom_role_policy")
apps = policy.get("trusted_check_app_slugs")
labels = policy.get("trusted_label_actors")
exemptions = policy.get("trusted_human_exemption_actors")
if not all(isinstance(value, list) and all(isinstance(item, str) and item for item in value)
for value in (apps, labels, exemptions)):
trusted = _extract_trust_policy_lists(policy)
if trusted is None:
return verdict("invalid_trust_policy")
apps, labels, exemptions = trusted
if not apps or not labels or not exemptions:
return verdict("trust_policy_unprovisioned")
required = {"schema_version", "pull_number", "head_sha", "publisher", "applicability", "label_authorization", "focused_tests", "agent_events"}
Expand Down Expand Up @@ -78,10 +107,23 @@ def verify(payload: Any, policy: Any, head_sha: str, pull_number: int) -> dict[s


def main() -> int:
if len(sys.argv) == 3 and sys.argv[1] == "--missing-publication":
policy = json.loads(Path(sys.argv[2]).read_text())
result = missing_publication(policy)
print(json.dumps(result, sort_keys=True))
return 0 if result["conclusion"] != "failure" else 1
if len(sys.argv) != 5:
raise SystemExit("usage: verifier REPORT POLICY HEAD_SHA PULL_NUMBER")
raise SystemExit(
"usage: verifier REPORT POLICY HEAD_SHA PULL_NUMBER "
"or verifier --missing-publication POLICY"
)
report, policy, head, pull = sys.argv[1:]
result = verify(json.loads(Path(report).read_text()), json.loads(Path(policy).read_text()), head, int(pull))
result = verify(
json.loads(Path(report).read_text()),
json.loads(Path(policy).read_text()),
head,
int(pull),
)
print(json.dumps(result, sort_keys=True))
return 0 if result["conclusion"] == "success" else 1

Expand Down
24 changes: 23 additions & 1 deletion tests/unit/test_agent_completion_enforcement.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import unittest

from scripts.ci.agent_completion_enforcement import verify
from scripts.ci.agent_completion_enforcement import missing_publication, verify


HEAD = "a" * 40
Expand Down Expand Up @@ -30,3 +30,25 @@ def test_error_cannot_be_erased(self):
def test_untrusted_label_blocks(self):
body = report(); body["label_authorization"]["applied_by"] = "agent"
self.assertEqual(verify(body, POLICY, HEAD, 9)["reason"], "untrusted_label_authorization")

def test_missing_publication_is_neutral_when_policy_unprovisioned(self):
policy = dict(
POLICY,
trusted_check_app_slugs=[],
trusted_label_actors=[],
trusted_human_exemption_actors=[],
)
result = missing_publication(policy)
self.assertEqual(result["conclusion"], "neutral")
self.assertEqual(result["reason"], "missing_publication_policy_unprovisioned")

def test_missing_publication_fails_when_policy_is_provisioned(self):
policy = dict(
POLICY,
trusted_check_app_slugs=["agent-lock-trusted"],
trusted_label_actors=["maintainer"],
trusted_human_exemption_actors=["maintainer"],
)
result = missing_publication(policy)
self.assertEqual(result["conclusion"], "failure")
self.assertEqual(result["reason"], "missing_trusted_publication")
Loading