Security status report SKILL

.agents/skills/security-status-report/SKILL.md

@@ -0,0 +1,42 @@
+---
+name: security-status-report
+description: Generates a security status report based on docs/threats.json by spinning up sub-agents for each threat to compute a quality score.
+---
+
+# Task
+Generate a security status report by evaluating threats listed in `docs/threats.json`.
+
+# Workflow
+
+1. Copy `.agents/skills/security-status-report/scripts/template_dispatch.py` to `.agents/scratch/security-status-report/template_dispatch.py`.
+2. Complete the TODO in `.agents/scratch/security-status-report/template_dispatch.py`, meeting the following requirements:
+  a. Iterate over each threat from `docs/threats.json` to produce a list of invocations that matches your tool for invoking sub-agents.
+  b. Each invocation must address a single threat, use the below prompt, and instruct the sub-agent to output the correct schema.
+3. Execute the script using `run_command` from the repository root, specifying `.agents/scratch/security-status-report/subagents.json` as the output file argument (`python3 .agents/scratch/security-status-report/template_dispatch.py .agents/scratch/security-status-report/subagents.json`).
+4. Read `.agents/scratch/security-status-report/subagents.json` and copy its exact JSON array into your tool for invoking subagents. DO NOT manually craft or bypass the invocations. Run ALL generated sub-agents concurrently in a single tool call. You already updated the script in step 2 to output exactly what you need, so no modifications to the output should be necessary unless you made a mistake.
+5. Wait for all sub-agents to complete.
+6. Run `python3 .agents/skills/security-status-report/scripts/compile_report.py docs/threats.json .agents/scratch/security-status-report .agents/scratch/security-status-report/final.json` to produce the final report.
+7. Run `python3 .agents/skills/security-status-report/scripts/render_chart.py .agents/scratch/security-status-report/final.json .agents/scratch/security-status-report/chart.png` to render a bar chart of the scores.
+
+## Prompt template for each sub-agent that should be used in the script
+
+You are a security reviewer evaluating the following specific threat:
+
+{INSERT EACH THREAT JSON VERBATIM HERE}
+
+- Focus on this threat only.
+- Review the entire repo.
+- Produce a gut-feel "quality score" based on the current security posture of the repo with respect to that threat.
+- Output your results using the following schema, by writing them to `.agents/scratch/security-status-report/{threat_id}.json`,
+  where `{threat_id}` matches the id in the threat json you were initially provided.
+
+```json
+{
+  "threat_id": "<threat_id from input>",
+  "threat": "<threat text from input>",
+  "quality": <Decimal between 0 (no effective mitigation) and 1 (perfectly mitigated).>,
+  "strengths": "<Specific positive code/design mechanisms responsible for the score.>",
+  "weaknesses": "<Specific negative code/design mechanisms responsible for the score.>",
+  "citations": ["<repo-relative/path/to/file1.go>"]
+}
+```

.agents/skills/security-status-report/scripts/compile_report.py

@@ -0,0 +1,101 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json
+import os
+
+def compile_report(threats_json_path, results_dir, output_path):
+    with open(threats_json_path, 'r') as f:
+        threats = json.load(f)
+
+    final_report = []
+    succeeded_count = 0
+    failed_ids = []
+    total_quality = 0.0
+
+    for t in threats:
+        threat_id = t.get("threat_id", "unknown")
+        result_file = os.path.join(results_dir, f"{threat_id}.json")
+
+        if os.path.exists(result_file):
+            try:
+                with open(result_file, 'r') as rf:
+                    res = json.load(rf)
+
+                # Check threat_id matching
+                if res.get("threat_id") and res.get("threat_id") != threat_id:
+                    t["error"] = f"Mismatched threat_id in result file: expected {threat_id}, got {res.get('threat_id')}"
+                else:
+                    # Validate quality score presence, type, and range
+                    if "quality" not in res:
+                        t["error"] = "Result JSON missing 'quality' score"
+                    else:
+                        try:
+                            score = float(res["quality"])
+                            if not (0.0 <= score <= 1.0):
+                                t["error"] = f"Quality score out of bounds [0.0, 1.0]: {res['quality']}"
+                            else:
+                                t["quality"] = score
+                        except (ValueError, TypeError):
+                            t["error"] = f"Invalid non-numeric quality score: {res['quality']}"
+
+                    # Copy strengths and weaknesses if no quality error
+                    if "error" not in t:
+                        if "strengths" in res:
+                            t["strengths"] = str(res["strengths"])
+                        if "weaknesses" in res:
+                            t["weaknesses"] = str(res["weaknesses"])
+
+                        # Validate and normalize citations schema
+                        if "citations" in res:
+                            c = res["citations"]
+                            if isinstance(c, list):
+                                t["citations"] = [str(item) for item in c]
+                            elif isinstance(c, str):
+                                t["citations"] = [c]
+                            else:
+                                t["citations"] = []
+            except Exception as e:
+                t["error"] = f"Failed to parse agent JSON: {e}"
+        else:
+            t["error"] = "The evaluation sub-agent timed out or failed to produce a valid JSON."
+
+        if "error" in t:
+            failed_ids.append(threat_id)
+        else:
+            succeeded_count += 1
+            total_quality += t.get("quality", 0.0)
+
+        final_report.append(t)
+
+    output_dir = os.path.dirname(output_path)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+    with open(output_path, 'w') as f:
+        json.dump(final_report, f, indent=2)
+
+    total_count = len(final_report)
+    avg_quality = (total_quality / succeeded_count) if succeeded_count > 0 else 0.0
+
+    print(f"Report compiled successfully to {output_path}")
+    print(f"Summary: {total_count} total threats | {succeeded_count} succeeded | {len(failed_ids)} failed")
+    if succeeded_count > 0:
+        print(f"Average Quality Score: {avg_quality:.2f}")
+
+    if failed_ids:
+        print(f"Warning: The following {len(failed_ids)} threat(s) failed evaluation: {', '.join(failed_ids)}", file=sys.stderr)
+
+if __name__ == '__main__':
+    compile_report(sys.argv[1], sys.argv[2], sys.argv[3])

.agents/skills/security-status-report/scripts/render_chart.py

@@ -0,0 +1,95 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json
+import os
+
+try:
+    import matplotlib
+    matplotlib.use('Agg')
+    import matplotlib.pyplot as plt
+except ImportError:
+    print("Error: 'matplotlib' is required to render the chart. Please install it using 'pip install matplotlib'.", file=sys.stderr)
+    sys.exit(1)
+
+def render_chart(final_report_path, output_png_path):
+    with open(final_report_path, 'r') as f:
+        data = json.load(f)
+
+    # Sort data by threat_id (e.g. T-01, T-02, ...)
+    def sort_key(item):
+        tid = item.get("threat_id", "")
+        if tid.startswith("T-") and tid[2:].isdigit():
+            return (0, int(tid[2:]), "")
+        return (1, 0, tid)
+
+    sorted_data = sorted(data, key=sort_key)
+
+    threat_ids = [item.get("threat_id", f"UNKNOWN-{i+1:02d}") for i, item in enumerate(sorted_data)]
+
+    scores = []
+    colors = []
+    errors = []
+
+    for item in sorted_data:
+        raw_score = item.get("quality", 0.0)
+        is_error = "error" in item
+        errors.append(is_error)
+
+        try:
+            score = float(raw_score) if not is_error else 0.0
+        except (ValueError, TypeError):
+            score = 0.0
+        score = max(0.0, min(1.0, score))
+        scores.append(score)
+
+        if is_error:
+            colors.append("#dc2626")  # Red for error / timeout
+        elif score >= 0.8:
+            colors.append("#16a34a")  # Green
+        elif score >= 0.5:
+            colors.append("#d97706")  # Orange
+        else:
+            colors.append("#dc2626")  # Red
+
+    fig, ax = plt.subplots(figsize=(14, 6))
+    ax.bar(threat_ids, scores, color=colors, width=0.6)
+
+    # Annotate errors with vertical "ERROR" text above x-axis
+    for i, err in enumerate(errors):
+        if err:
+            ax.text(i, 0.02, "ERROR", rotation=90, ha='center', va='bottom', fontsize=8, fontweight='bold', color='#dc2626')
+
+    ax.set_ylim(0.0, 1.0)
+    ax.set_ylabel("Quality Score (0.0 - 1.0)", fontsize=12, fontweight='bold')
+    ax.set_xlabel("Threat ID", fontsize=12, fontweight='bold')
+    ax.set_title("Substrate Security Threat Posture Scores", fontsize=16, fontweight='bold', pad=15)
+
+    # Standard matplotlib way to rotate tick labels cleanly
+    plt.xticks(rotation=90, fontsize=9, fontweight='bold')
+    ax.grid(axis='y', linestyle='--', alpha=0.5)
+
+    plt.tight_layout()
+    output_dir = os.path.dirname(output_png_path)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+    plt.savefig(output_png_path, dpi=150)
+    plt.close()
+    print(f"Chart rendered successfully to {output_png_path}")
+
+if __name__ == '__main__':
+    report_file = sys.argv[1] if len(sys.argv) > 1 else ".agents/scratch/security-status-report/final.json"
+    output_png = sys.argv[2] if len(sys.argv) > 2 else ".agents/scratch/security-status-report/chart.png"
+    render_chart(report_file, output_png)

.agents/skills/security-status-report/scripts/template_dispatch.py

@@ -0,0 +1,66 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Template script for formatting threats into agent dispatch specifications.
+The main agent should complete this script to output a valid JSON array
+matching the 'Subagents' parameter of its invoke_subagent tool.
+"""
+import sys
+import json
+import os
+
+def generate_dispatch_payload(output_path):
+    with open('docs/threats.json', 'r') as f:
+        threats = json.load(f)
+
+    subagents = []
+
+    for t in threats:
+        prompt = f'''You are a security reviewer evaluating the following specific threat:
+
+{json.dumps(t, indent=2)}
+
+- Focus on this threat only.
+- Review the entire repo.
+- Produce a gut-feel "quality score" based on the current security posture of the repo with respect to that threat.
+- Output your results using the following schema, by writing them to .agents/scratch/security-status-report/{t["threat_id"]}.json,
+  where {t["threat_id"]} matches the id in the threat json you were initially provided.
+
+```json
+{{
+  "threat_id": "<threat_id from input>",
+  "threat": "<threat text from input>",
+  "quality": <Decimal between 0 (no effective mitigation) and 1 (perfectly mitigated).>,
+  "strengths": "<Specific positive code/design mechanisms responsible for the score.>",
+  "weaknesses": "<Specific negative code/design mechanisms responsible for the score.>",
+  "citations": ["<repo-relative/path/to/file1.go>"]
+}}
+```'''
+        # TODO: Agent, update this part to ensure it matches the correct schema for you to invoke sub-agents via a tool call.
+        subagents.append({
+            "Prompt": prompt,
+            "Role": "Security Reviewer",
+            "TypeName": "self",
+            "Workspace": "inherit"
+        })
+
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, 'w') as f:
+        json.dump(subagents, f, indent=2)
+    print(f"Dispatch payload written successfully to {output_path}")
+
+if __name__ == '__main__':
+    output_path = sys.argv[1] if len(sys.argv) > 1 else ".agents/scratch/security-status-report/subagents.json"
+    generate_dispatch_payload(output_path)

.gitignore

@@ -53,3 +53,6 @@ Thumbs.db
 
 # Stray local build outputs (go build ./tools/... without -o)
 /validate-image-cache
+
+# Substrate agent workspace scratchpads
+.agents/scratch/