v2.7.2–2.7.9: AI/ML detectors, impact-rich verification, mobile UX, resilience, caching — and eight bug fixes

.env.example

@@ -151,3 +151,40 @@ MAX_SOURCEMAP_SOURCES=200
 # response for missing/weak security headers, version disclosure, and insecure
 # cookies. Pure analysis of a response the target already serves.
 SCAN_HTTP_POSTURE=true
+
+# Inline SSR state decoding (v2.7.6). Server-rendered apps embed bootstrap state
+# in the HTML (__NEXT_DATA__, __NUXT__, __INITIAL_STATE__, __APOLLO_STATE__).
+# The raw HTML is already scanned, so a plainly-embedded secret is caught either
+# way; this decodes the JSON so a credential mangled by \uXXXX escaping is still
+# recognised. Local decode only — no extra requests, scan stays passive.
+# Asset caching (v2.7.7). Sends If-None-Match / If-Modified-Since on re-scans.
+# An asset that is BOTH unchanged (304) AND produced no finding last time is
+# skipped entirely; one that previously had a finding is always refetched so the
+# finding cannot silently vanish from a report. No response body is ever cached
+# — only validators, a content hash, and a clean/dirty flag — because a client's
+# JavaScript can contain live credentials. Cache entries expire after 24h.
+# Client reports redact the credential by default (sk_abc…******…3fc4 (51 chars)).
+# A report gets emailed, forwarded and archived, so writing a live secret into
+# one turns the deliverable itself into a second exposure — and the RoE we ask
+# clients to sign promises redaction. The redacted form stays greppable, so a
+# developer can still identify exactly which key to rotate. Set true only when
+# you deliberately need the full value in the artefact.
+REPORT_FULL_SECRETS=false
+
+ASSET_CACHE=true
+
+SCAN_INLINE_JSON=true
+MAX_INLINE_JSON_BYTES=2000000
+
+# ── Scan politeness / resilience (v2.7.5) ──────────────────────────────────
+# Retries use exponential backoff with equal jitter so concurrent workers do not
+# all retry on the same tick (a thundering herd against a host that just asked
+# for relief). RETRY_MAX_BACKOFF caps any single wait, including a server's
+# Retry-After.
+RETRY_MAX_BACKOFF=30.0
+# Adaptive per-host throttle: when a host answers 429/503 we start pacing
+# requests to THAT host only, growing by THROTTLE_STEP per signal up to
+# THROTTLE_MAX_DELAY, and decaying as it recovers. Costs nothing while a host is
+# healthy. Raise THROTTLE_STEP to be gentler on fragile client infrastructure.
+THROTTLE_STEP=0.5
+THROTTLE_MAX_DELAY=5.0

.github/workflows/ci.yml

@@ -37,6 +37,13 @@ jobs:
       - name: Run test suite
         run: pytest
 
+      # A scanner's whole value is that its findings can be trusted, so a
+      # precision regression is release-blocking, not a warning. This measures
+      # the deterministic detection layer against the labelled corpus and fails
+      # the build if precision or recall drops.
+      - name: Detection quality gate (precision/recall)
+        run: make bench
+
   docker-build:
     runs-on: ubuntu-latest
     steps:

README.md

@@ -231,7 +231,7 @@ http://<raspberry-pi-ip>:8000
 
 ---
 
-## Secret Patterns Detected (44)
+## Secret Patterns Detected (63)
 
 Every pattern carries a **severity** and a **CWE** id, and only fires after passing a Shannon-entropy
 filter (so obvious placeholders like `YOUR_API_KEY_HERE` are dropped before the AI stage).

backend/bench/corpus.py

@@ -24,6 +24,7 @@
 
 _ALNUM = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
 _UPPER = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+_HEX   = "0123456789abcdef"
 
 
 def _rand(n: int, seed: str, alphabet: str = _ALNUM) -> str:
@@ -52,6 +53,12 @@ def is_positive(self) -> bool:
         return self.expect is not None
 
 
+def _hex(n: int, seed: str = "hex") -> str:
+    """Deterministic lowercase-hex string — the shape ElevenLabs / OpenRouter /
+    LangSmith keys and SHA digests use."""
+    return _rand(n, seed + str(n), _HEX)
+
+
 # ── POSITIVES — a real (synthetic) secret is planted; the layer must catch it ──
 _POSITIVES: list[Sample] = [
     Sample("aws-access-key",
@@ -113,4 +120,47 @@ def is_positive(self) -> bool:
     Sample("css-color-hash", ".btn { color: #a3f2c1; background: #1b2e4d; border: #ff00aa; }", None),
 ]
 
-CORPUS: list[Sample] = _POSITIVES + _NEGATIVES
+# ── v2.7.7 · AI/ML provider detectors + inline-SSR decoding ─────────────────
+# A quality gate that does not cover the newest detectors is not a quality gate.
+# These exercise every pattern added in v2.7.2 plus the v2.7.6 inline-JSON path.
+
+_AI_POSITIVES: list[Sample] = [
+    Sample("elevenlabs", f'window.REACT_APP_ELEVENLABS_API_KEY = "sk_{_hex(48)}";',
+           "ElevenLabs API Key"),
+    Sample("groq", f'client = Groq(api_key="gsk_{_rand(52, "groq")}")', "Groq API Key"),
+    Sample("huggingface", f'HF_TOKEN = "hf_{_rand(37, "hf")}"', "Hugging Face Access Token"),
+    Sample("replicate", f'REPLICATE_API_TOKEN="r8_{_rand(40, "rep")}"', "Replicate API Token"),
+    Sample("perplexity", f'PPLX_KEY = "pplx-{_rand(48, "pplx")}"', "Perplexity API Key"),
+    Sample("xai", f'XAI_API_KEY="xai-{_rand(80, "xai")}"', "xAI API Key"),
+    Sample("openrouter", f'k = "sk-or-v1-{_hex(64)}"', "OpenRouter API Key"),
+    Sample("langsmith", f'LANGCHAIN_API_KEY="lsv2_pt_{_hex(32)}_{_hex(10)}"',
+           "LangSmith API Key"),
+    Sample("pinecone", f'pc = Pinecone(api_key="pcsk_{_rand(60, "pc")}")', "Pinecone API Key"),
+    # v2.7.6: the credential is \uXXXX-escaped inside inline SSR state, so the
+    # raw-text pass cannot see it — only decoding the JSON recovers it.
+    Sample("inline-ssr-escaped",
+           '<script id="__NEXT_DATA__" type="application/json">{"props":{"k":"'
+           + ("sk-ant-" + _rand(40, "ant")).replace("-", r"\u002D") + '"}}</script>',
+           "Anthropic API Key"),
+]
+
+# Hard negatives: strings that structurally resemble the new detectors but are
+# not credentials. These are where false positives actually come from.
+_AI_NEGATIVES: list[Sample] = [
+    # sk_ + hex but the wrong length for ElevenLabs
+    Sample("sk-hex-short", f'id = "sk_{_hex(20)}"', None),
+    Sample("sk-hex-long", f'id = "sk_{_hex(72)}"', None),
+    # right prefix, obvious placeholder
+    Sample("elevenlabs-placeholder", 'ELEVENLABS_API_KEY = "sk_your_api_key_here"', None),
+    Sample("hf-placeholder", 'HF_TOKEN = "hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"', None),
+    # a 64-hex blob that is a digest, not an OpenRouter key
+    Sample("sha256-digest", f'integrity = "{_hex(64)}"', None),
+    # docs-style examples
+    Sample("groq-docs", 'api_key="gsk_YOUR_GROQ_KEY"', None),
+    Sample("openrouter-docs", 'Authorization: Bearer sk-or-v1-<your-key>', None),
+    # benign inline JSON must not produce anything
+    Sample("inline-json-benign",
+           '<script type="application/json">{"title":"Docs","items":[1,2,3]}</script>', None),
+]
+
+CORPUS: list[Sample] = _POSITIVES + _NEGATIVES + _AI_POSITIVES + _AI_NEGATIVES

backend/bench/run_bench.py

@@ -25,6 +25,8 @@
 
 from __future__ import annotations
 
+import os
+
 from bench.corpus import CORPUS, Sample
 from scanner import extract_secrets
 
@@ -96,5 +98,32 @@ def format_report(m: dict) -> str:
     return "\n".join(lines)
 
 
+# Quality floor enforced in CI. A detector change that drops below either bar
+# fails the build instead of silently shipping: a scanner's whole value is that
+# its findings can be trusted, so precision regressions are release-blocking.
+MIN_PRECISION = float(os.environ.get("BENCH_MIN_PRECISION", "1.0"))
+MIN_RECALL    = float(os.environ.get("BENCH_MIN_RECALL", "1.0"))
+
+
+def main() -> int:
+    m = evaluate()
+    print(format_report(m))
+    failures = []
+    if m["precision"] < MIN_PRECISION:
+        failures.append(f"precision {m['precision']:.3f} < required {MIN_PRECISION:.3f}")
+    if m["recall"] < MIN_RECALL:
+        failures.append(f"recall {m['recall']:.3f} < required {MIN_RECALL:.3f}")
+    if failures:
+        print("\nDETECTION QUALITY GATE FAILED:")
+        for f in failures:
+            print(f"  ✗ {f}")
+        print("\nEither fix the detector, or — if the corpus is wrong — update it "
+              "deliberately in the same commit and say why.")
+        return 1
+    print("\n  quality gate: PASS "
+          f"(precision >= {MIN_PRECISION:.2f}, recall >= {MIN_RECALL:.2f})")
+    return 0
+
+
 if __name__ == "__main__":  # pragma: no cover
-    print(format_report(evaluate()))
+    raise SystemExit(main())

backend/main.py

@@ -42,10 +42,11 @@
 # 3.12+, and uvicorn already selects the event loop itself via --loop (auto/uvloop),
 # so the speed-up is preserved without a module-level global side effect.
 
-from scanner import run_scan, ScanState
+from scanner import run_scan, ScanState, load_asset_cache, drain_asset_cache
 import orchestrator
 from storage import (
     init_db, save_scan, load_scans, load_scan, get_previous_scan_for_target,
+    get_asset_cache, save_asset_cache,
     mark_false_positive, unmark_false_positive, get_suppressed_fingerprints,
     list_false_positives,
 )
@@ -327,6 +328,10 @@ async def _run() -> None:
             )
             suppressed_fps = await get_suppressed_fingerprints(request.target_url)
 
+            # Prime the conditional-GET cache so unchanged, previously-clean
+            # assets can be skipped instead of re-downloaded.
+            load_asset_cache(await get_asset_cache(request.target_url))
+
             result = await run_scan(
                 target_url=request.target_url,
                 scan_id=scan_id,
@@ -340,6 +345,7 @@ async def _run() -> None:
             )
             _registry[scan_id]["meta"] = result
             await save_scan(scan_id, result)
+            await save_asset_cache(request.target_url, drain_asset_cache())
         except asyncio.CancelledError:
             logger.info("Scan %s task cancelled", scan_id)
             await manager.broadcast_scan(scan_id, {

backend/report.py

@@ -19,6 +19,7 @@
 import csv
 import html
 import io
+import os
 import json
 from datetime import datetime, timezone
 from pathlib import Path
@@ -45,7 +46,7 @@ def _tool_version() -> str:
                 return s.split("=", 1)[1].strip().strip('"').strip("'")
     except Exception:
         pass
-    return "2.7.1"
+    return "2.7.9"
 
 
 _TOOL_VERSION = _tool_version()
@@ -55,6 +56,29 @@ def _severity_of(finding: dict[str, Any]) -> str:
     return str(finding.get("severity", "MEDIUM")).upper()
 
 
+# Client reports get emailed, forwarded and archived. Writing a live credential
+# into one turns the deliverable itself into a second exposure — and the RoE we
+# ask clients to sign promises the opposite: "only the minimum evidence needed
+# to prove a finding, with sensitive data redacted".
+#
+# A redacted value still does its job: the developer can grep for the prefix and
+# identify exactly which key to rotate, without the report handing an attacker a
+# working credential. An operator who genuinely needs the full value (handing it
+# to the client's engineer for rotation) can opt in explicitly.
+REPORT_FULL_SECRETS = os.environ.get("REPORT_FULL_SECRETS", "false").lower() == "true"
+
+
+def redact_secret(value: str) -> str:
+    """Mask the middle of a credential, keeping enough to identify it."""
+    if not value:
+        return ""
+    if REPORT_FULL_SECRETS:
+        return value
+    if len(value) <= 12:
+        return "*" * len(value)
+    return f"{value[:6]}…{'*' * 6}…{value[-4:]}  ({len(value)} chars)"
+
+
 def _sort_key(finding: dict[str, Any]) -> tuple[int, int, str]:
     """Sort by severity (critical first), then AI confidence (high first),
     then secret type for stable ordering."""
@@ -153,7 +177,7 @@ def finding_row(f: dict[str, Any]) -> str:
           <td class="mono">{html.escape(f.get('source_url', f.get('target_url','')))}</td>
           <td>{f.get('confidence',0)}%</td>
           <td>{badge}{ver_badge(f)}</td>
-          <td class="mono small">{html.escape(f.get('raw_match',''))}</td>
+          <td class="mono small">{html.escape(redact_secret(f.get('raw_match','')))}</td>
           <td class="small">{html.escape(f.get('reason',''))}</td>
         </tr>"""
 
@@ -400,13 +424,13 @@ def generate_csv_report(scan: dict[str, Any]) -> str:
             f.get("source_url", f.get("target_url", "")),
             f.get("confidence", 0), "NEW" if f.get("is_new", True) else "RECURRING",
             f.get("verified", "disabled"), f.get("verified_detail", ""), f.get("impact", ""),
-            f.get("raw_match", ""), f.get("reason", ""), f.get("found_at", ""),
+            redact_secret(f.get("raw_match", "")), f.get("reason", ""), f.get("found_at", ""),
         ])
     for f in scan.get("needs_review_findings", []):
         writer.writerow([
             "NEEDS_REVIEW", _severity_of(f), f.get("cwe", ""), f.get("secret_type", ""),
             f.get("source_url", f.get("target_url", "")),
-            "", "", "", "", f.get("impact", ""), f.get("raw_match", ""),
+            "", "", "", "", f.get("impact", ""), redact_secret(f.get("raw_match", "")),
             f.get("reason", ""), f.get("found_at", ""),
         ])
     return buf.getvalue()