diff --git a/.env.example b/.env.example index d488f4d..374b8f0 100644 --- a/.env.example +++ b/.env.example @@ -157,20 +157,28 @@ SCAN_HTTP_POSTURE=true # The raw HTML is already scanned, so a plainly-embedded secret is caught either # way; this decodes the JSON so a credential mangled by \uXXXX escaping is still # recognised. Local decode only — no extra requests, scan stays passive. -# Asset caching (v2.7.7). Sends If-None-Match / If-Modified-Since on re-scans. -# An asset that is BOTH unchanged (304) AND produced no finding last time is -# skipped entirely; one that previously had a finding is always refetched so the -# finding cannot silently vanish from a report. No response body is ever cached -# — only validators, a content hash, and a clean/dirty flag — because a client's -# JavaScript can contain live credentials. Cache entries expire after 24h. # Client reports redact the credential by default (sk_abc…******…3fc4 (51 chars)). # A report gets emailed, forwarded and archived, so writing a live secret into # one turns the deliverable itself into a second exposure — and the RoE we ask # clients to sign promises redaction. The redacted form stays greppable, so a # developer can still identify exactly which key to rotate. Set true only when # you deliberately need the full value in the artefact. +# +# This affects generated reports ONLY. The dashboard, the WebSocket stream and +# the REST API always mask, with no opt-out — those are screen-sharing and +# screenshot surfaces, not deliverables an operator chose to produce. REPORT_FULL_SECRETS=false +# Asset caching (v2.7.7). Sends If-None-Match / If-Modified-Since on re-scans. +# A *terminal* asset (JS bundle, source map) that is BOTH unchanged (304) AND +# produced no finding last time is skipped entirely; one that previously had a +# finding is always refetched so the finding cannot silently vanish from a +# report. Crawled HTML pages are never skipped, even when unchanged (v2.8.0): a +# page is a link graph, and skipping its body means never discovering the +# bundles it references — which turned a re-scan of an unchanged site into a +# false all-clear. No response body is ever cached — only validators, a content +# hash, and a clean/dirty flag — because a client's JavaScript can contain live +# credentials. Cache entries expire after 24h. ASSET_CACHE=true SCAN_INLINE_JSON=true diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e1080..63ad2b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,82 @@ All notable changes to SecretNode are documented here. This project adheres to [Semantic Versioning](https://semver.org/). +## [2.8.0] — Artifact review: the credential stops at the API boundary + +Driven by a side-by-side read of the artifacts from a real v2.7.9 deep scan — the dashboard modal, +the Discord alert, and the SARIF/CSV/HTML exports of the same three findings. v2.7.9 had fixed +redaction in the *reports*; comparing the surfaces showed how much of the same job was left undone +everywhere else, and turned up two correctness bugs that had nothing to do with display. + +### Fixed + +- **A re-scan of an unchanged site reported CLEAN while the credential was still exposed.** The + worst bug in this release, and the only one that produces a wrong *answer* rather than a wrong + *presentation*. The asset cache treated a `304 Not Modified` on the root page as "unchanged and + previously clean, skip" — but an HTML page is a link graph, not just something to grep. Skipping + its body meant never parsing its `' + requested: list[tuple[str, bool]] = [] + + class _Client: + async def get(self, url, headers=None, **kw): + conditional = bool(headers and "If-None-Match" in headers) + requested.append((url, conditional)) + # Server says "unchanged" whenever we revalidate. + return FakeResponse(304) if conditional else FakeResponse(200, page, {"etag": '"v1"'}) + + scanner.load_asset_cache({"https://x.example": {"etag": '"v1"', "was_clean": True}}) + try: + url, body = await scanner.fetch_url( + _Client(), "https://x.example", asyncio.Semaphore(1), + allow_cache_skip=False, + ) + finally: + scanner.load_asset_cache({}) + + assert body != scanner.CACHED_CLEAN, "a crawled page must never be cache-skipped" + assert "app.js" in body, "the body is needed to discover linked assets" + assert requested[0][1] is True, "the conditional GET is still sent — bandwidth is still saved" + assert len(requested) == 2, "a 304 on a page triggers an immediate unconditional refetch" + + +@pytest.mark.asyncio +async def test_a_terminal_asset_is_still_cache_skipped(): + """The optimisation must survive the fix: JS bundles are leaves in the graph, + nothing is discovered from skipping them, so they still skip.""" + import asyncio + + class _Client: + async def get(self, url, headers=None, **kw): + return FakeResponse(304) + + scanner.load_asset_cache({"https://x.example/app.js": {"etag": '"v1"', "was_clean": True}}) + try: + _, body = await scanner.fetch_url( + _Client(), "https://x.example/app.js", asyncio.Semaphore(1), + ) + assert body == scanner.CACHED_CLEAN + assert scanner.cached_clean_count() == 1 + finally: + scanner.load_asset_cache({}) + + +@pytest.mark.asyncio +async def test_a_scan_does_not_inherit_the_previous_scans_cache_tally(): + """`_asset_cache_hits` is module state. A deep-scan host never primes the + cache, so without an explicit reset it reported the *previous* scan's hit + count as its own coverage.""" + scanner.load_asset_cache({}) + scanner._asset_cache_hits.update({"https://old/a.js", "https://old/b.js"}) + + async def _noop_spider(*a, **kw): + return [] + + import unittest.mock as mock + with mock.patch.object(scanner, "spider_target", _noop_spider): + result = await scanner.run_scan(target_url="https://new.example", scan_id="s2") + + assert result["assets_cached"] == 0, "stale tally leaked into a fresh scan" + assert result["assets_scanned"] == 0 + + +def test_deep_scan_starts_from_an_empty_asset_cache(): + src = (Path(__file__).resolve().parent.parent / "orchestrator.py").read_text(encoding="utf-8") + assert "scanner.load_asset_cache({})" in src + + +def test_report_leads_with_coverage_not_downloads(): + scan = { + "target_url": "https://x", "scan_id": "s", "status": "clean", + "assets_fetched": 0, "assets_cached": 9, "assets_scanned": 9, + "raw_findings": 0, "confirmed_findings": [], "needs_review_findings": [], + } + html = report.generate_html_report(scan) + assert "Assets analysed 9" in html + assert "unchanged since the previous scan" in html + assert "0 asset(s) were analysed" not in html + + +def test_report_falls_back_for_records_written_before_the_cache_counters(): + """Scans already in SQLite have no assets_scanned key.""" + scan = { + "target_url": "https://x", "scan_id": "s", "status": "clean", + "assets_fetched": 4, "raw_findings": 0, + "confirmed_findings": [], "needs_review_findings": [], + } + assert "Assets analysed 4" in report.generate_html_report(scan) + + +def test_deep_scan_rolls_up_cache_coverage(): + d = _deep_result() + d.scans[0].update({"assets_fetched": 7, "assets_cached": 3, "assets_scanned": 10}) + d.scans[1].update({"assets_fetched": 5, "assets_cached": 0, "assets_scanned": 5}) + dd = d.to_dict() + assert dd["assets_cached"] == 3 + assert dd["assets_scanned"] == 15 + assert dd["totals"]["assets_scanned"] == 15 + + +def test_env_example_documents_report_full_secrets(): + env = Path(__file__).resolve().parent.parent.parent / ".env.example" + assert "REPORT_FULL_SECRETS" in env.read_text(encoding="utf-8") + + +assert os.environ.setdefault("SECRETNODE_API_KEY", "test-key") diff --git a/backend/version.py b/backend/version.py new file mode 100644 index 0000000..bcd65dc --- /dev/null +++ b/backend/version.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +""" +Single source of truth for the SecretNode version string. + +Every surface that stamps a version — the API banner, client reports, the +Discord alerter — reads it from here, which in turn reads pyproject.toml. The +alternative (a literal per module) is how the Discord alerter ended up still +announcing "v2.4.0" long after the tool shipped 2.7.9: nothing fails when a +hardcoded string goes stale, it just quietly misreports which build produced a +finding, and a client correlating an alert against a report sees two tools. +""" + +from __future__ import annotations + +from pathlib import Path + +# Used only when pyproject.toml is unreadable (an odd install layout, a +# stripped container). Keep it in step with pyproject on every release. +_FALLBACK_VERSION = "2.8.0" + + +def read_version() -> str: + """Parse `version = "x.y.z"` out of the project's pyproject.toml.""" + try: + pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml" + for line in pyproject.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if s.startswith("version") and "=" in s: + return s.split("=", 1)[1].strip().strip('"').strip("'") + except Exception: + pass + return _FALLBACK_VERSION + + +TOOL_VERSION = read_version() diff --git a/frontend/index.html b/frontend/index.html index 3d6b449..a2bd1fe 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -384,6 +384,7 @@ .badge-critical { background: rgba(255,59,59,0.15); color: #ff3b3b; border: 1px solid rgba(255,59,59,0.3); } .badge-high { background: rgba(255,123,0,0.15); color: #ff7b00; border: 1px solid rgba(255,123,0,0.3); } .badge-medium { background: rgba(255,215,0,0.12); color: #ffd700; border: 1px solid rgba(255,215,0,0.3); } + .badge-low { background: rgba(49,130,206,0.12);color: #63b3ed; border: 1px solid rgba(49,130,206,0.3);} .badge-info { background: rgba(99,179,237,0.12);color: #63b3ed; border: 1px solid rgba(99,179,237,0.3);} /* ── Stats cards ── */ @@ -816,12 +817,29 @@ } const ts = () => new Date().toISOString().replace('T',' ').slice(0,19); - function severityFromType(type) { - const crit = ['AWS Access Key','AWS Secret Access Key','GitHub Personal Access Token','Stripe Secret Key','Private Key Block']; - const high = ['Google Cloud API Key','Slack Webhook','GitHub OAuth Token','SendGrid API Key','Twilio Auth Token','Heroku API Key','Shopify Access Token','Mailgun API Key','JWT Token']; - if (crit.includes(type)) return 'critical'; - if (high.includes(type)) return 'high'; - return 'medium'; + // Severity comes from the backend, which reads it off the pattern registry + // and applies the public-by-design downgrade. This used to be re-derived + // here from a hardcoded list of 14 secret types; every detector added since + // — the whole AI/ML provider family included — fell through to MEDIUM, so + // the dashboard showed MEDIUM for a finding the SARIF and CSV both called + // HIGH. Never re-derive a value the server already computed. + const VALID_SEVERITIES = ['critical', 'high', 'medium', 'low', 'info']; + + function severityOf(finding) { + const sev = String(finding?.severity || '').toLowerCase(); + return VALID_SEVERITIES.includes(sev) ? sev : 'medium'; + } + + // Defence in depth: the API masks `raw_match` before it leaves the server, + // but the dashboard must never render a value that arrived unmasked (an + // older scan replayed from SQLite, a stale service behind a new frontend). + // If it carries no mask marker, mask it here before it hits the DOM. + function maskSecret(value) { + const v = String(value || ''); + if (!v) return ''; + if (v.includes('*') || v.includes('…')) return v; // already masked + if (v.length <= 12) return '*'.repeat(v.length); + return `${v.slice(0, 6)}…${'*'.repeat(6)}…${v.slice(-4)} (${v.length} chars)`; } function confidenceColor(c) { @@ -954,7 +972,7 @@ const tbody = $('findings-tbody'); $('empty-row')?.remove(); - const sev = severityFromType(finding.secret_type); + const sev = severityOf(finding); const cc = confidenceColor(finding.confidence); const time = finding.found_at ? new Date(finding.found_at).toLocaleTimeString() @@ -1026,7 +1044,7 @@ const f = state.allFindings[index]; if (!f) return; - const sev = severityFromType(f.secret_type); + const sev = severityOf(f); const mc = $('modal-content'); mc.innerHTML = `
@@ -1064,8 +1082,9 @@
${escapeHtml(f.context_snippet||'')}
-
MATCHED VALUE (PARTIAL)
- ${escapeHtml((f.raw_match||'').slice(0,60))}… +
MATCHED VALUE (REDACTED)
+ ${escapeHtml(maskSecret(f.raw_match))} +
Enough to identify which key to rotate — the full value is never rendered here. Download a report to hand the value to the owner.
`; $('modal-overlay').classList.add('show'); diff --git a/pyproject.toml b/pyproject.toml index 349205d..f0016c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "secretnode" -version = "2.7.9" +version = "2.8.0" description = "Real-time passive attack-surface scanner for credential-leak detection, with a two-tier Gemini AI validation engine (google-genai), live verification, source-map mining, WAF-resilient fetching, SARIF export, and a live dashboard." readme = "README.md" requires-python = ">=3.11"