Phase 7: Resilience & test hardening

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 110
+extend-ignore = E203
+exclude = .git,__pycache__,dashboard,.chroma

.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  backend:
+    runs-on: ubuntu-latest
+    env:
+      # tests mock all external boundaries; these only satisfy import-time env lookups
+      DATABASE_URL: postgresql://ci:ci@localhost:5432/ci
+      OPENROUTER_API_KEY: ci-dummy
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - run: pip install -e core/ pytest jsonschema black flake8
+      - run: black --check --line-length 110 core sandbox
+      - run: flake8 core sandbox
+      - run: pytest core/tests -q
+
+  dashboard:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: dashboard
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: dashboard/package-lock.json
+      # ponytail: npm install, not npm ci — Windows-generated lockfiles omit optional
+      # wasm-binding deps (@emnapi/*) that Linux npm ci then rejects
+      - run: npm install --no-audit --no-fund
+      - run: npm run lint
+      - run: npm test

README.md

@@ -1,5 +1,7 @@
 # Sentinel
 
+![CI](https://github.com/Lushenwar/Sentinel/actions/workflows/ci.yml/badge.svg)
+
 <!-- DEMO GIF: record per docs/demo-script.md and replace this comment with:
 ![Sentinel demo](docs/demo.gif) -->
 

contract/pipeline_schema.json

@@ -5,7 +5,7 @@
   "required": ["incident_id", "status", "trigger"],
   "properties": {
     "incident_id": { "type": "string", "pattern": "^inc_" },
-    "status": { "type": "string", "enum": ["triggered", "triaging", "resolved"] },
+    "status": { "type": "string", "enum": ["triggered", "triaging", "degraded", "resolved"] },
     "trigger": {
       "type": "object",
       "required": ["source", "alert_name", "timestamp", "error_signature"],
@@ -47,8 +47,23 @@
         "impact_assessment": {
           "type": "object",
           "properties": {
-            "error_rate_delta_pct": { "type": "number" },
-            "estimated_affected_users": { "type": "integer" }
+            "error_rate_delta_pct": { "type": ["number", "null"] },
+            "estimated_affected_users": { "type": ["integer", "null"] }
+          }
+        },
+        "degraded": { "type": "boolean" },
+        "degraded_reason": { "type": "string" },
+        "raw_commits": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "commit_hash": { "type": "string" },
+              "author": { "type": "string" },
+              "timestamp": { "type": "string" },
+              "subject": { "type": "string" },
+              "diff": { "type": "string" }
+            }
           }
         }
       }

core/db.py

@@ -1,15 +1,18 @@
-import os, json
+import json
+import os
 import psycopg2
 from psycopg2.extras import RealDictCursor
 from dotenv import load_dotenv
 
 load_dotenv()
 DSN = os.environ["DATABASE_URL"]
 
+
 # ponytail: new connection per call; add psycopg2.pool.ThreadedConnectionPool if throughput matters
 def _conn():
     return psycopg2.connect(DSN)
 
+
 def init_schema():
     conn = _conn()
     with conn:
@@ -36,6 +39,7 @@ def init_schema():
             """)
     conn.close()
 
+
 def save_incident(incident: dict):
     conn = _conn()
     with conn:
@@ -57,6 +61,7 @@ def save_incident(incident: dict):
             )
     conn.close()
 
+
 def get_incident(incident_id: str) -> dict | None:
     conn = _conn()
     with conn.cursor(cursor_factory=RealDictCursor) as cur:
@@ -65,6 +70,7 @@ def get_incident(incident_id: str) -> dict | None:
     conn.close()
     return dict(row) if row else None
 
+
 def list_incidents() -> list:
     conn = _conn()
     with conn.cursor(cursor_factory=RealDictCursor) as cur:
@@ -73,6 +79,7 @@ def list_incidents() -> list:
     conn.close()
     return [dict(r) for r in rows]
 
+
 def update_status(incident_id: str, status: str):
     conn = _conn()
     with conn:
@@ -83,16 +90,19 @@ def update_status(incident_id: str, status: str):
             )
     conn.close()
 
+
 def update_diagnostics(incident_id: str, diagnostics: dict):
     conn = _conn()
     with conn:
         with conn.cursor() as cur:
             cur.execute(
-                "UPDATE incidents SET diagnostics = %s, status = 'triaging', updated_at = NOW() WHERE id = %s",
+                "UPDATE incidents SET diagnostics = %s, status = 'triaging', updated_at = NOW() "
+                "WHERE id = %s",
                 (json.dumps(diagnostics), incident_id),
             )
     conn.close()
 
+
 def update_postmortem(incident_id: str, postmortem: str):
     conn = _conn()
     with conn:

core/main.py

@@ -4,13 +4,15 @@
 from . import db, orchestrator
 from .services import vector_store, git_client
 
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     db.init_schema()
     n = vector_store.ingest_runbooks(orchestrator.RUNBOOKS_DIR)
     print(f"[core] runbooks ingested: {n}")
     yield
 
+
 app = FastAPI(title="Sentinel Core", lifespan=lifespan)
 
 # ponytail: dev-only origin; lock down to the deployed dashboard origin before shipping
@@ -21,23 +23,27 @@ async def lifespan(app: FastAPI):
     allow_headers=["*"],
 )
 
+
 @app.post("/alert")
 def receive_alert(payload: dict, background_tasks: BackgroundTasks) -> dict:
     incident = orchestrator.handle_alert(payload)
     background_tasks.add_task(orchestrator.run_diagnostics, incident["incident_id"], payload)
     return incident
 
+
 @app.get("/incidents")
 def list_incidents() -> list:
     return db.list_incidents()
 
+
 @app.get("/incidents/{incident_id}")
 def get_incident(incident_id: str) -> dict:
     inc = db.get_incident(incident_id)
     if not inc:
         raise HTTPException(404, "incident not found")
     return inc
 
+
 @app.post("/incidents/{incident_id}/resolve")
 def resolve_incident(incident_id: str, background_tasks: BackgroundTasks) -> dict:
     inc = db.get_incident(incident_id)
@@ -46,6 +52,7 @@ def resolve_incident(incident_id: str, background_tasks: BackgroundTasks) -> dic
     background_tasks.add_task(orchestrator.resolve_incident, incident_id)
     return {**inc, "status": "resolving"}
 
+
 @app.patch("/incidents/{incident_id}/postmortem")
 def save_postmortem(incident_id: str, payload: dict) -> dict:
     if "postmortem" not in payload:
@@ -56,6 +63,7 @@ def save_postmortem(incident_id: str, payload: dict) -> dict:
     db.update_postmortem(incident_id, payload["postmortem"])
     return db.get_incident(incident_id)
 
+
 @app.get("/incidents/{incident_id}/commits/{commit_hash}/diff")
 def get_commit_diff(incident_id: str, commit_hash: str) -> dict:
     diff = git_client.get_commit_diff(orchestrator.REPO_PATH, commit_hash)

core/orchestrator.py

@@ -7,6 +7,7 @@
 REPO_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))  # Sentinel root
 RUNBOOKS_DIR = os.path.join(REPO_PATH, "sandbox", "runbooks")
 
+
 def handle_alert(alert_data: dict) -> dict:
     ts = datetime.now(timezone.utc).strftime("%Y_%m%d_%H%M%S")
     incident = {
@@ -20,6 +21,7 @@ def handle_alert(alert_data: dict) -> dict:
     print(f"[orchestrator] {incident['incident_id']} created -> triggered")
     return incident
 
+
 def run_diagnostics(incident_id: str, alert_data: dict):
     """Runs in a background task after the alert response is sent."""
     print(f"[orchestrator] {incident_id} -> triaging")
@@ -29,12 +31,17 @@ def run_diagnostics(incident_id: str, alert_data: dict):
         diffs = git_client.get_recent_diffs(REPO_PATH, alert_data["timestamp"])
         print(f"[orchestrator] {incident_id} -- {len(diffs)} commits in window")
 
-        ranked = llm_analyzer.rank_suspect_commits(diffs, alert_data) if diffs else []
-        print(f"[orchestrator] {incident_id} -- LLM ranked {len(ranked)} suspects")
-
         runbooks = vector_store.find_matching_runbooks(alert_data.get("error_signature", ""))
         print(f"[orchestrator] {incident_id} -- {len(runbooks)} runbooks matched")
 
+        degraded_reason = None
+        try:
+            ranked = llm_analyzer.rank_suspect_commits(diffs, alert_data) if diffs else []
+            print(f"[orchestrator] {incident_id} -- LLM ranked {len(ranked)} suspects")
+        except llm_analyzer.LLMUnavailable as e:
+            ranked, degraded_reason = [], str(e)
+            print(f"[orchestrator] {incident_id} -- LLM unavailable, degrading: {e}")
+
         diagnostics = {
             "suspect_commits": ranked,
             "matched_runbooks": runbooks,
@@ -43,11 +50,23 @@ def run_diagnostics(incident_id: str, alert_data: dict):
                 "estimated_affected_users": None,
             },
         }
+        if degraded_reason:
+            diagnostics["degraded"] = True
+            diagnostics["degraded_reason"] = (
+                f"Diagnostic unavailable — manual investigation required ({degraded_reason})"
+            )
+            # raw material for the manual investigation the flag asks for
+            diagnostics["raw_commits"] = diffs
+
         db.update_diagnostics(incident_id, diagnostics)
-        alert_dt = datetime.fromisoformat(alert_data["timestamp"].replace("Z", "+00:00"))
-        elapsed = (datetime.now(timezone.utc) - alert_dt).total_seconds()
-        print(f"[orchestrator] {incident_id} -> diagnostics saved")
-        print(f"[metrics] {incident_id} alert_to_suspects_s={elapsed:.1f}")
+        if degraded_reason:
+            db.update_status(incident_id, "degraded")
+            print(f"[orchestrator] {incident_id} -> degraded (terminal)")
+        else:
+            alert_dt = datetime.fromisoformat(alert_data["timestamp"].replace("Z", "+00:00"))
+            elapsed = (datetime.now(timezone.utc) - alert_dt).total_seconds()
+            print(f"[orchestrator] {incident_id} -> diagnostics saved")
+            print(f"[metrics] {incident_id} alert_to_suspects_s={elapsed:.1f}")
 
         try:
             notifier.post_incident_to_slack(db.get_incident(incident_id))
@@ -58,11 +77,20 @@ def run_diagnostics(incident_id: str, alert_data: dict):
         print(f"[orchestrator] {incident_id} diagnostics failed: {e}")
         db.update_status(incident_id, "triggered")  # revert so it can be retried
 
+
 def resolve_incident(incident_id: str) -> dict:
     """Runs in a background task once an incident is marked resolved."""
     t0 = time.monotonic()
     incident = db.get_incident(incident_id)
-    draft = postmortem.generate_postmortem(incident)
+    try:
+        draft = postmortem.generate_postmortem(incident)
+    except llm_analyzer.LLMUnavailable as e:
+        draft = (
+            "# Postmortem unavailable\n\n"
+            f"Automatic generation failed ({e}). Manual write-up required — "
+            "trigger and diagnostics data remain on the incident record."
+        )
+        print(f"[orchestrator] {incident_id} -- postmortem LLM unavailable, degrading: {e}")
     db.update_postmortem(incident_id, draft)
     print(f"[orchestrator] {incident_id} -> resolved, postmortem generated")
     print(f"[metrics] {incident_id} resolve_to_postmortem_s={time.monotonic() - t0:.1f}")

core/services/git_client.py

@@ -1,6 +1,6 @@
 import subprocess
-from datetime import datetime, timezone, timedelta
-from pathlib import Path
+from datetime import datetime, timedelta
+
 
 def get_recent_diffs(repo_path: str, alert_ts: str, window_minutes: int = 60) -> list[dict]:
     """Return commits with diffs in [alert_ts - window_minutes, alert_ts]."""
@@ -10,7 +10,11 @@ def get_recent_diffs(repo_path: str, alert_ts: str, window_minutes: int = 60) ->
 
     log = subprocess.run(
         ["git", "log", "--format=%H|%ae|%aI|%s", f"--since={since}", f"--until={until}"],
-        cwd=repo_path, capture_output=True, text=True, encoding="utf-8", errors="replace",
+        cwd=repo_path,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
     )
     if log.returncode != 0 or not log.stdout.strip():
         return []
@@ -24,23 +28,34 @@ def get_recent_diffs(repo_path: str, alert_ts: str, window_minutes: int = 60) ->
 
         diff = subprocess.run(
             ["git", "show", "--stat", "--patch", full_hash],
-            cwd=repo_path, capture_output=True, text=True, encoding="utf-8", errors="replace",
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+        commits.append(
+            {
+                "commit_hash": full_hash[:7],
+                "full_hash": full_hash,
+                "author": author,
+                "timestamp": ts,
+                "subject": subject,
+                "diff": diff.stdout[:3000],  # ponytail: truncated for LLM context; raise if diffs are large
+            }
         )
-        commits.append({
-            "commit_hash": full_hash[:7],
-            "full_hash": full_hash,
-            "author": author,
-            "timestamp": ts,
-            "subject": subject,
-            "diff": diff.stdout[:3000],  # ponytail: truncated for LLM context; raise if diffs are large
-        })
 
     return commits
 
+
 def get_commit_diff(repo_path: str, commit_hash: str) -> str:
     """Full untruncated diff for a single commit, for dashboard display."""
     result = subprocess.run(
         ["git", "show", "--stat", "--patch", commit_hash],
-        cwd=repo_path, capture_output=True, text=True, encoding="utf-8", errors="replace",
+        cwd=repo_path,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
     )
     return result.stdout if result.returncode == 0 else ""