From 036348cd2bc8459a2d6db0587f07266e1a025f8d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 09:48:32 +0000 Subject: [PATCH] Artifact review: stop the credential at the API boundary (v2.8.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Driven by reading the artifacts of a real v2.7.9 deep scan side by side — the dashboard modal, the Discord alert, and the SARIF/CSV/HTML of the same three findings. v2.7.9 fixed redaction in the reports; comparing the surfaces showed how much of that job was left undone everywhere else, and turned up two correctness bugs that had nothing to do with display. A re-scan of an unchanged site reported CLEAN while the credential was still exposed. The asset cache treated a 304 on the root page as "unchanged and previously clean, skip", but a page is a link graph — skipping its body meant never parsing its ' + requested: list[tuple[str, bool]] = [] + + class _Client: + async def get(self, url, headers=None, **kw): + conditional = bool(headers and "If-None-Match" in headers) + requested.append((url, conditional)) + # Server says "unchanged" whenever we revalidate. + return FakeResponse(304) if conditional else FakeResponse(200, page, {"etag": '"v1"'}) + + scanner.load_asset_cache({"https://x.example": {"etag": '"v1"', "was_clean": True}}) + try: + url, body = await scanner.fetch_url( + _Client(), "https://x.example", asyncio.Semaphore(1), + allow_cache_skip=False, + ) + finally: + scanner.load_asset_cache({}) + + assert body != scanner.CACHED_CLEAN, "a crawled page must never be cache-skipped" + assert "app.js" in body, "the body is needed to discover linked assets" + assert requested[0][1] is True, "the conditional GET is still sent — bandwidth is still saved" + assert len(requested) == 2, "a 304 on a page triggers an immediate unconditional refetch" + + +@pytest.mark.asyncio +async def test_a_terminal_asset_is_still_cache_skipped(): + """The optimisation must survive the fix: JS bundles are leaves in the graph, + nothing is discovered from skipping them, so they still skip.""" + import asyncio + + class _Client: + async def get(self, url, headers=None, **kw): + return FakeResponse(304) + + scanner.load_asset_cache({"https://x.example/app.js": {"etag": '"v1"', "was_clean": True}}) + try: + _, body = await scanner.fetch_url( + _Client(), "https://x.example/app.js", asyncio.Semaphore(1), + ) + assert body == scanner.CACHED_CLEAN + assert scanner.cached_clean_count() == 1 + finally: + scanner.load_asset_cache({}) + + +@pytest.mark.asyncio +async def test_a_scan_does_not_inherit_the_previous_scans_cache_tally(): + """`_asset_cache_hits` is module state. A deep-scan host never primes the + cache, so without an explicit reset it reported the *previous* scan's hit + count as its own coverage.""" + scanner.load_asset_cache({}) + scanner._asset_cache_hits.update({"https://old/a.js", "https://old/b.js"}) + + async def _noop_spider(*a, **kw): + return [] + + import unittest.mock as mock + with mock.patch.object(scanner, "spider_target", _noop_spider): + result = await scanner.run_scan(target_url="https://new.example", scan_id="s2") + + assert result["assets_cached"] == 0, "stale tally leaked into a fresh scan" + assert result["assets_scanned"] == 0 + + +def test_deep_scan_starts_from_an_empty_asset_cache(): + src = (Path(__file__).resolve().parent.parent / "orchestrator.py").read_text(encoding="utf-8") + assert "scanner.load_asset_cache({})" in src + + +def test_report_leads_with_coverage_not_downloads(): + scan = { + "target_url": "https://x", "scan_id": "s", "status": "clean", + "assets_fetched": 0, "assets_cached": 9, "assets_scanned": 9, + "raw_findings": 0, "confirmed_findings": [], "needs_review_findings": [], + } + html = report.generate_html_report(scan) + assert "Assets analysed 9" in html + assert "unchanged since the previous scan" in html + assert "0 asset(s) were analysed" not in html + + +def test_report_falls_back_for_records_written_before_the_cache_counters(): + """Scans already in SQLite have no assets_scanned key.""" + scan = { + "target_url": "https://x", "scan_id": "s", "status": "clean", + "assets_fetched": 4, "raw_findings": 0, + "confirmed_findings": [], "needs_review_findings": [], + } + assert "Assets analysed 4" in report.generate_html_report(scan) + + +def test_deep_scan_rolls_up_cache_coverage(): + d = _deep_result() + d.scans[0].update({"assets_fetched": 7, "assets_cached": 3, "assets_scanned": 10}) + d.scans[1].update({"assets_fetched": 5, "assets_cached": 0, "assets_scanned": 5}) + dd = d.to_dict() + assert dd["assets_cached"] == 3 + assert dd["assets_scanned"] == 15 + assert dd["totals"]["assets_scanned"] == 15 + + +def test_env_example_documents_report_full_secrets(): + env = Path(__file__).resolve().parent.parent.parent / ".env.example" + assert "REPORT_FULL_SECRETS" in env.read_text(encoding="utf-8") + + +assert os.environ.setdefault("SECRETNODE_API_KEY", "test-key") diff --git a/backend/version.py b/backend/version.py new file mode 100644 index 0000000..bcd65dc --- /dev/null +++ b/backend/version.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +""" +Single source of truth for the SecretNode version string. + +Every surface that stamps a version — the API banner, client reports, the +Discord alerter — reads it from here, which in turn reads pyproject.toml. The +alternative (a literal per module) is how the Discord alerter ended up still +announcing "v2.4.0" long after the tool shipped 2.7.9: nothing fails when a +hardcoded string goes stale, it just quietly misreports which build produced a +finding, and a client correlating an alert against a report sees two tools. +""" + +from __future__ import annotations + +from pathlib import Path + +# Used only when pyproject.toml is unreadable (an odd install layout, a +# stripped container). Keep it in step with pyproject on every release. +_FALLBACK_VERSION = "2.8.0" + + +def read_version() -> str: + """Parse `version = "x.y.z"` out of the project's pyproject.toml.""" + try: + pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml" + for line in pyproject.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if s.startswith("version") and "=" in s: + return s.split("=", 1)[1].strip().strip('"').strip("'") + except Exception: + pass + return _FALLBACK_VERSION + + +TOOL_VERSION = read_version() diff --git a/frontend/index.html b/frontend/index.html index 3d6b449..a2bd1fe 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -384,6 +384,7 @@ .badge-critical { background: rgba(255,59,59,0.15); color: #ff3b3b; border: 1px solid rgba(255,59,59,0.3); } .badge-high { background: rgba(255,123,0,0.15); color: #ff7b00; border: 1px solid rgba(255,123,0,0.3); } .badge-medium { background: rgba(255,215,0,0.12); color: #ffd700; border: 1px solid rgba(255,215,0,0.3); } + .badge-low { background: rgba(49,130,206,0.12);color: #63b3ed; border: 1px solid rgba(49,130,206,0.3);} .badge-info { background: rgba(99,179,237,0.12);color: #63b3ed; border: 1px solid rgba(99,179,237,0.3);} /* ── Stats cards ── */ @@ -816,12 +817,29 @@ } const ts = () => new Date().toISOString().replace('T',' ').slice(0,19); - function severityFromType(type) { - const crit = ['AWS Access Key','AWS Secret Access Key','GitHub Personal Access Token','Stripe Secret Key','Private Key Block']; - const high = ['Google Cloud API Key','Slack Webhook','GitHub OAuth Token','SendGrid API Key','Twilio Auth Token','Heroku API Key','Shopify Access Token','Mailgun API Key','JWT Token']; - if (crit.includes(type)) return 'critical'; - if (high.includes(type)) return 'high'; - return 'medium'; + // Severity comes from the backend, which reads it off the pattern registry + // and applies the public-by-design downgrade. This used to be re-derived + // here from a hardcoded list of 14 secret types; every detector added since + // — the whole AI/ML provider family included — fell through to MEDIUM, so + // the dashboard showed MEDIUM for a finding the SARIF and CSV both called + // HIGH. Never re-derive a value the server already computed. + const VALID_SEVERITIES = ['critical', 'high', 'medium', 'low', 'info']; + + function severityOf(finding) { + const sev = String(finding?.severity || '').toLowerCase(); + return VALID_SEVERITIES.includes(sev) ? sev : 'medium'; + } + + // Defence in depth: the API masks `raw_match` before it leaves the server, + // but the dashboard must never render a value that arrived unmasked (an + // older scan replayed from SQLite, a stale service behind a new frontend). + // If it carries no mask marker, mask it here before it hits the DOM. + function maskSecret(value) { + const v = String(value || ''); + if (!v) return ''; + if (v.includes('*') || v.includes('…')) return v; // already masked + if (v.length <= 12) return '*'.repeat(v.length); + return `${v.slice(0, 6)}…${'*'.repeat(6)}…${v.slice(-4)} (${v.length} chars)`; } function confidenceColor(c) { @@ -954,7 +972,7 @@ const tbody = $('findings-tbody'); $('empty-row')?.remove(); - const sev = severityFromType(finding.secret_type); + const sev = severityOf(finding); const cc = confidenceColor(finding.confidence); const time = finding.found_at ? new Date(finding.found_at).toLocaleTimeString() @@ -1026,7 +1044,7 @@ const f = state.allFindings[index]; if (!f) return; - const sev = severityFromType(f.secret_type); + const sev = severityOf(f); const mc = $('modal-content'); mc.innerHTML = `
@@ -1064,8 +1082,9 @@
${escapeHtml(f.context_snippet||'')}
-
MATCHED VALUE (PARTIAL)
- ${escapeHtml((f.raw_match||'').slice(0,60))}… +
MATCHED VALUE (REDACTED)
+ ${escapeHtml(maskSecret(f.raw_match))} +
Enough to identify which key to rotate — the full value is never rendered here. Download a report to hand the value to the owner.
`; $('modal-overlay').classList.add('show'); diff --git a/pyproject.toml b/pyproject.toml index 349205d..f0016c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "secretnode" -version = "2.7.9" +version = "2.8.0" description = "Real-time passive attack-surface scanner for credential-leak detection, with a two-tier Gemini AI validation engine (google-genai), live verification, source-map mining, WAF-resilient fetching, SARIF export, and a live dashboard." readme = "README.md" requires-python = ">=3.11"