From 6ab492adaead3e5b483fa7d638efc716d4fbebae Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 18:11:53 +0000 Subject: [PATCH 01/19] Fix false negative: class-aware entropy gating for structural detectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Shannon-entropy floor (MIN_ENTROPY_THRESHOLD=3.5) was applied uniformly to every detector. Structural/provider patterns (AKIA…, ghp_…, sk_live_…, PEM, fixed-format hex/UUID tokens) are high-precision by shape, but a genuinely low-entropy yet well-formed live key — e.g. an AWS key ID at ~3.27 bits/char — was silently dropped before it ever reached AI validation. That is a false negative, the worst failure mode for a secret scanner. Entropy handling is now class-aware: • generic keyword=value catch-all keeps the full 3.5 bar (needs a randomness signal to stay quiet); • structural detectors clear only a low anti-degenerate floor (MIN_STRUCTURAL_ENTROPY=2.5) that still rejects obvious junk such as "AKIAAAAAAAAAAAAAAAAA" (~0.6 bits) while catching modest-entropy real keys. SecretPattern gains an entropy_gated flag (default False; True only on the generic pattern). Adds MIN_STRUCTURAL_ENTROPY config + .env.example docs. New tests lock in both directions (low-entropy structural key detected; degenerate junk still rejected; generic gate unchanged). Precision/recall stays 1.000/1.000; suite 187 -> 191, ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- .env.example | 7 +++++- CHANGELOG.md | 12 +++++++++ backend/scanner.py | 22 ++++++++++++++++- backend/tests/test_scanner.py | 46 +++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 8d2ce90..ef6f33a 100644 --- a/.env.example +++ b/.env.example @@ -57,8 +57,13 @@ PORT=8000 CONCURRENCY_LIMIT=20 # Seconds per HTTP request before timeout. FETCH_TIMEOUT=20.0 -# Shannon-entropy floor (bits/char). Raise to 4.0 to cut Gemini cost / false matches. +# Shannon-entropy floor (bits/char) for the GENERIC keyword=value catch-all. +# Raise to 4.0 to cut Gemini cost / false matches. MIN_ENTROPY_THRESHOLD=3.5 +# Low anti-degenerate entropy floor for STRUCTURAL/provider detectors (AKIA…, ghp_…, +# sk_live_…). Only rejects obvious junk (e.g. all-identical chars); keep it well below +# MIN_ENTROPY_THRESHOLD so genuinely modest-entropy live keys are not dropped. +MIN_STRUCTURAL_ENTROPY=2.5 # Skip JS assets larger than this many bytes (default 5 MB). MAX_ASSET_BYTES=5242880 # Only surface AI-confirmed findings at or above this confidence (0-100). diff --git a/CHANGELOG.md b/CHANGELOG.md index d42657b..c95672b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to SecretNode are documented here. This project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Fixed +- **False-negative: structural keys wrongly entropy-gated.** The Shannon-entropy floor + (`MIN_ENTROPY_THRESHOLD=3.5`) was applied uniformly to every detector, silently dropping + genuinely low-entropy but well-formed provider keys (e.g. an AWS key ID at ~3.27 bits) before + they ever reached AI validation — the worst failure mode for a scanner. Entropy is now + class-aware: the *generic* keyword=value catch-all keeps the full 3.5 bar, while + *structural/provider* detectors (AKIA…, ghp_…, sk_live_…, PEM, fixed-format tokens) only clear a + low anti-degenerate floor (`MIN_STRUCTURAL_ENTROPY=2.5`) that still rejects obvious junk like + `AKIAAAAAAAAAAAAAAAAA`. Precision/recall stays 1.000/1.000; suite **187 → 191**. + ## [2.6.0] — Detection quality, safety & attack-surface breadth A measured capability pass grounded in a fresh audit vs 2026 secret-scanning SOTA diff --git a/backend/scanner.py b/backend/scanner.py index 1d98b4c..305f772 100644 --- a/backend/scanner.py +++ b/backend/scanner.py @@ -89,6 +89,11 @@ def _env_float(name: str, default: float) -> float: RETRY_ATTEMPTS = _env_int("RETRY_ATTEMPTS", 3) RETRY_BACKOFF_BASE = _env_float("RETRY_BACKOFF_BASE", 2.0) MIN_ENTROPY_THRESHOLD = _env_float("MIN_ENTROPY_THRESHOLD", 3.5) +MIN_STRUCTURAL_ENTROPY = _env_float("MIN_STRUCTURAL_ENTROPY", 2.5) # low anti-degenerate + # floor for high-precision structural detectors: rejects + # obvious junk (e.g. "AKIAAAAAAAAAAAAAAAAA", ~0.6 bits) while + # still catching genuinely modest-entropy live keys that the + # full generic bar would wrongly drop (false-negative guard). CONTEXT_WINDOW_CHARS = _env_int("CONTEXT_WINDOW_CHARS", 120) MAX_ASSET_BYTES = _env_int("MAX_ASSET_BYTES", 5 * 1024 * 1024) # 5 MB GEMINI_CONFIDENCE_MIN = _env_int("GEMINI_CONFIDENCE_MIN", 80) @@ -124,6 +129,16 @@ class SecretPattern: severity: str = "HIGH" cwe: str = "CWE-798" # Use of Hard-coded Credentials remediation: str = _DEFAULT_REMEDIATION + # Entropy handling differs by detector class: + # • Structural/provider detectors (AKIA…, ghp_…, sk_live_…, PEM blocks, + # fixed-format hex/UUID tokens) are high-precision by shape. They get + # only a LOW anti-degenerate floor (MIN_STRUCTURAL_ENTROPY) that rejects + # obvious junk like "AKIAAAAAAAAAAAAAAAAA" while still catching genuinely + # modest-entropy live keys — because gating these on the full generic bar + # silently drops real credentials (a false negative, the worst failure). + # • The generic keyword=value catch-all matches loosely and needs the full + # MIN_ENTROPY_THRESHOLD randomness signal to stay quiet; it opts in below. + entropy_gated: bool = False @dataclass @@ -314,6 +329,7 @@ def _meta(self) -> "SecretPattern": ), description="Generic credential assignment", severity="MEDIUM", + entropy_gated=True, # loose keyword=value match — entropy keeps it quiet ), SecretPattern( name="Mailgun API Key", @@ -1251,7 +1267,11 @@ def _scan_text( if is_benign_placeholder(raw_value): continue entropy = shannon_entropy(raw_value) - if not passes_entropy_check(raw_value): + # Generic keyword=value catch-all must clear the full randomness bar; + # structural detectors only need to clear a low anti-degenerate floor + # so genuinely modest-entropy live keys are not silently dropped. + floor = MIN_ENTROPY_THRESHOLD if pattern.entropy_gated else MIN_STRUCTURAL_ENTROPY + if entropy < floor: continue start = max(0, match.start() - CONTEXT_WINDOW_CHARS) end = min(len(text), match.end() + CONTEXT_WINDOW_CHARS) diff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.py index af1dd16..e4a8ec2 100644 --- a/backend/tests/test_scanner.py +++ b/backend/tests/test_scanner.py @@ -180,6 +180,52 @@ def test_no_false_positive_on_placeholder(self): assert findings == [] or all(f.secret_type != "AWS Access Key" for f in findings) +class TestEntropyGatingPolicy: + """Entropy gating is a false-positive control for the *generic* keyword=value + catch-all only. Structural/provider detectors are high-precision by shape and + must NOT be entropy-gated — otherwise a genuinely low-entropy but well-formed + live key (e.g. an AWS key ID whose 16 chars happen to be low-entropy) is + silently dropped: a false negative, the worst failure mode for a scanner.""" + + def test_low_entropy_structural_key_is_detected(self): + # A correctly-shaped AWS key ID whose entropy is *below* the threshold. + low = "AKIA6218374A3D288737" + assert scanner.shannon_entropy(low) < scanner.MIN_ENTROPY_THRESHOLD + body = f'const cfg = {{ key: "{low}" }};' + findings = scanner.extract_secrets( + "scan1", "https://example.com", "https://example.com/app.js", body + ) + assert "AWS Access Key" in [f.secret_type for f in findings] + + def test_degenerate_structural_key_is_rejected(self): + # All-identical chars => degenerate junk, not a real key. The low + # structural floor still rejects it even though it matches the AWS shape. + junk = "AKIA" + "A" * 16 + assert scanner.shannon_entropy(junk) < scanner.MIN_STRUCTURAL_ENTROPY + body = f'k = "{junk}"' + findings = scanner.extract_secrets( + "scan1", "https://example.com", "https://example.com/app.js", body + ) + assert "AWS Access Key" not in [f.secret_type for f in findings] + + def test_aws_pattern_is_not_entropy_gated(self): + assert scanner.PATTERN_BY_NAME["AWS Access Key"].entropy_gated is False + + def test_generic_pattern_stays_entropy_gated(self): + assert scanner.PATTERN_BY_NAME["Generic High-Entropy Secret"].entropy_gated is True + + def test_generic_low_entropy_value_is_still_dropped(self): + # Loose keyword=value with a low-entropy (non-placeholder) value must + # still be filtered — the generic catch-all keeps its entropy gate. + low_val = "a" * 24 + assert scanner.shannon_entropy(low_val) < scanner.MIN_ENTROPY_THRESHOLD + body = f'api_key = "{low_val}"' + findings = scanner.extract_secrets( + "scan1", "https://example.com", "https://example.com/app.js", body + ) + assert "Generic High-Entropy Secret" not in [f.secret_type for f in findings] + + class TestNeedsReviewSentinel: def test_sentinel_is_negative(self): # Must never collide with a real 0-100 confidence value From 13dc564f49fbd1a5748a00854d2aaa0e11fd2bd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 18:33:24 +0000 Subject: [PATCH 02/19] Fix false negative: route AI-dismissed structural matches to manual review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bucketing sent a finding to Confirmed only when is_valid AND confidence >= threshold, and to manual review only when AI validation was *unavailable* (sentinel). A structural/provider match the AI *actively* rejected with a real confidence matched neither bucket and was silently dropped — so a genuine live key the AI merely under-called on (e.g. a page that gave it no surrounding context) vanished with no trace. Add classify_validated() returning confirmed | review | drop: • AI unavailable (sentinel) -> review (unchanged) • is_valid and confidence >= min -> confirmed (unchanged) • structural match NOT confidently dismissed by the AI -> review (new: shape-anchored keys get human eyes instead of deletion) • everything else, incl. any generic catch-all 'no' -> drop (preserves aggressive generic filtering / 'no false positives in Confirmed') Report: broaden the manual-review section heading and the SARIF note to cover both reasons; the per-row Note already carries the AI's reasoning. Adds 6 classifier tests. Suite 191 -> 197, ruff clean, precision/recall unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- backend/report.py | 4 ++-- backend/scanner.py | 34 ++++++++++++++++++++++++++-------- backend/tests/test_scanner.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/backend/report.py b/backend/report.py index fde435a..6b5deb7 100644 --- a/backend/report.py +++ b/backend/report.py @@ -349,7 +349,7 @@ def _posture_row(p: dict[str, Any]) -> str: {posture_html} -

Flagged for Manual Review (AI validation unavailable)

+

Flagged for Manual Review (AI unavailable, or a structural match it could not confidently clear)

{review_html} @@ -491,7 +491,7 @@ def generate_sarif_report(scan: dict[str, Any]) -> str: impact = str(f.get("impact", "") or "") msg = ( vprefix + f"{secret_type} ({severity}) detected. " - f"{'AI validation unavailable — manual review required. ' if is_review else ''}" + f"{'Manual review required — see note. ' if is_review else ''}" f"{('Impact: ' + impact + ' ') if impact else ''}" f"{f.get('reason','')}" ) diff --git a/backend/scanner.py b/backend/scanner.py index 305f772..d5dcc79 100644 --- a/backend/scanner.py +++ b/backend/scanner.py @@ -1737,6 +1737,29 @@ def check(self) -> None: # Master Scan Orchestrator # ───────────────────────────────────────────────────────────────────────────── +def classify_validated(v: "ValidatedFinding") -> str: + """Route a validated finding to exactly one of: 'confirmed', 'review', 'drop'. + + The critical rule is the last one: a *structural/provider* match (high-precision + by shape — AKIA…, ghp_…, sk_live_…, PEM) that the AI did **not confidently + dismiss** is sent to manual review, never silently dropped. Without this, a real + live key the AI merely under-called on (e.g. because a page gave it no + surrounding context) would vanish with no trace — a false negative, the worst + failure mode for a scanner. The generic keyword=value catch-all keeps the old + aggressive behaviour (an AI 'no' there is trusted and dropped), preserving the + 'no false positives in Confirmed' promise.""" + if v.confidence == NEEDS_REVIEW_SENTINEL: + return "review" # AI unavailable — human decides + if v.is_valid and v.confidence >= GEMINI_CONFIDENCE_MIN: + return "confirmed" + meta = PATTERN_BY_NAME.get(v.raw.secret_type) + structural = meta is not None and not meta.entropy_gated + ai_confidently_dismissed = (not v.is_valid) and v.confidence >= GEMINI_CONFIDENCE_MIN + if structural and not ai_confidently_dismissed: + return "review" # shape-anchored + AI not sure it's fake → human confirms + return "drop" + + async def run_scan( target_url: str, scan_id: str | None = None, @@ -1930,14 +1953,9 @@ async def _validate_one(f: RawFinding) -> ValidatedFinding: reason=f"Unexpected validation error: {v}. Manual review required.", )) - confirmed: list[ValidatedFinding] = [ - v for v in validated - if v.is_valid and v.confidence >= GEMINI_CONFIDENCE_MIN - ] - needs_review: list[ValidatedFinding] = [ - v for v in validated - if v.confidence == NEEDS_REVIEW_SENTINEL - ] + _routed = [(classify_validated(v), v) for v in validated] + confirmed: list[ValidatedFinding] = [v for b, v in _routed if b == "confirmed"] + needs_review: list[ValidatedFinding] = [v for b, v in _routed if b == "review"] # ── Suppress known false positives ────────────────────────────── if suppressed_fingerprints: diff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.py index e4a8ec2..41db67d 100644 --- a/backend/tests/test_scanner.py +++ b/backend/tests/test_scanner.py @@ -226,6 +226,41 @@ def test_generic_low_entropy_value_is_still_dropped(self): assert "Generic High-Entropy Secret" not in [f.secret_type for f in findings] +class TestClassifyValidated: + """A structural match the AI does not *confidently* dismiss must go to manual + review, never be silently dropped (false-negative guard). The generic catch-all + keeps aggressive filtering so 'no false positives in Confirmed' still holds.""" + + def _vf(self, secret_type, is_valid, confidence): + raw = scanner.RawFinding( + scan_id="s", target_url="https://e.com", source_url="https://e.com/a.js", + secret_type=secret_type, raw_match="AKIA6218374A3D288737", + context_snippet="x", entropy=3.27, + ) + return scanner.ValidatedFinding(raw=raw, is_valid=is_valid, confidence=confidence, reason="r") + + def test_sentinel_goes_to_review(self): + vf = self._vf("AWS Access Key", False, scanner.NEEDS_REVIEW_SENTINEL) + assert scanner.classify_validated(vf) == "review" + + def test_valid_high_confidence_confirmed(self): + assert scanner.classify_validated(self._vf("AWS Access Key", True, 95)) == "confirmed" + + def test_structural_ai_uncertain_rejection_goes_to_review(self): + # AI says not-valid but only 55% sure => a shape-anchored key must not vanish. + assert scanner.classify_validated(self._vf("AWS Access Key", False, 55)) == "review" + + def test_structural_valid_but_low_confidence_goes_to_review(self): + assert scanner.classify_validated(self._vf("AWS Access Key", True, 60)) == "review" + + def test_structural_ai_confident_rejection_dropped(self): + assert scanner.classify_validated(self._vf("AWS Access Key", False, 95)) == "drop" + + def test_generic_ai_rejection_dropped_even_if_uncertain(self): + # The generic catch-all trusts an AI 'no' and drops it, regardless of confidence. + assert scanner.classify_validated(self._vf("Generic High-Entropy Secret", False, 55)) == "drop" + + class TestNeedsReviewSentinel: def test_sentinel_is_negative(self): # Must never collide with a real 0-100 confidence value From faca7cd119e8adaa359769e9ee80761df4b7535e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 18:33:45 +0000 Subject: [PATCH 03/19] docs: record AI-dismissed-structural fix in changelog Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c95672b..db659cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,14 @@ All notable changes to SecretNode are documented here. This project adheres to class-aware: the *generic* keyword=value catch-all keeps the full 3.5 bar, while *structural/provider* detectors (AKIA…, ghp_…, sk_live_…, PEM, fixed-format tokens) only clear a low anti-degenerate floor (`MIN_STRUCTURAL_ENTROPY=2.5`) that still rejects obvious junk like - `AKIAAAAAAAAAAAAAAAAA`. Precision/recall stays 1.000/1.000; suite **187 → 191**. + `AKIAAAAAAAAAAAAAAAAA`. Precision/recall stays 1.000/1.000. +- **False-negative: AI-dismissed structural matches silently dropped.** A finding was routed to + manual review only when AI validation was *unavailable*; a structural/provider match the AI + *actively* rejected with a real confidence matched no bucket and was discarded — so a live key the + AI merely under-called on (e.g. lacking page context) vanished with no trace. New + `classify_validated()` sends any structural match the AI does **not confidently dismiss** to + manual review instead of dropping it; the generic catch-all keeps aggressive filtering, so the + "no false positives in Confirmed" promise holds. Suite **187 → 197**. ## [2.6.0] — Detection quality, safety & attack-surface breadth From 71d9a9e462578faea8ef265c5f23d9f02bd5d264 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 18:51:04 +0000 Subject: [PATCH 04/19] Add passive subdomain enumeration via Certificate Transparency (deep-ASM slice 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First discovery layer of the passive attack-surface pipeline: given a domain, expand it into its known subdomain surface from Certificate Transparency (crt.sh) using public data only — no packet is ever sent to the target, so it runs before an engagement is signed. New backend/recon.py: • extract_registrable_domain() — normalise URL / host[:port] / bare domain to the registrable domain, with a two-label public-suffix table (incl. .co.uk, .com.au, and Bangladesh .*.bd). Returns None for IP literals and single-label inputs (nothing to enumerate). • parse_crtsh_json() — pure, deterministic parse of a crt.sh response into sorted, deduped, in-scope hostnames; strips wildcards, rejects emails and look-alike suffixes (evil.example.com.attacker.net). • enumerate_subdomains_ct() — async CT query that FAILS CLOSED: any network/parse error yields an empty result + error string, never an exception that could abort a scan. CLI: `python cli.py --subdomains` prints discovered subdomains as JSON (accepts a bare domain since it never fetches the target). Config: CRTSH_URL, SUBDOMAIN_ENUM_TIMEOUT, MAX_SUBDOMAINS (+ .env.example). 18 new tests (parsing, normalisation, fail-closed, limit); network mocked. Suite 197 -> 215, ruff clean. Passive/authorized posture preserved. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- .env.example | 9 ++ CHANGELOG.md | 9 ++ backend/cli.py | 31 ++++++ backend/recon.py | 198 ++++++++++++++++++++++++++++++++++++ backend/tests/test_recon.py | 133 ++++++++++++++++++++++++ 5 files changed, 380 insertions(+) create mode 100644 backend/recon.py create mode 100644 backend/tests/test_recon.py diff --git a/.env.example b/.env.example index ef6f33a..c3dfac4 100644 --- a/.env.example +++ b/.env.example @@ -64,6 +64,15 @@ MIN_ENTROPY_THRESHOLD=3.5 # sk_live_…). Only rejects obvious junk (e.g. all-identical chars); keep it well below # MIN_ENTROPY_THRESHOLD so genuinely modest-entropy live keys are not dropped. MIN_STRUCTURAL_ENTROPY=2.5 + +# ── Passive reconnaissance (subdomain enumeration) ────────────────────────── +# Certificate Transparency front-end used to enumerate subdomains passively. +# Discovery never contacts the target — only this CT source. +CRTSH_URL=https://crt.sh +# Seconds to wait for the CT-log query before failing closed (empty result). +SUBDOMAIN_ENUM_TIMEOUT=30 +# Safety cap on how many discovered subdomains to return. +MAX_SUBDOMAINS=500 # Skip JS assets larger than this many bytes (default 5 MB). MAX_ASSET_BYTES=5242880 # Only surface AI-confirmed findings at or above this confidence (0-100). diff --git a/CHANGELOG.md b/CHANGELOG.md index db659cf..6719fbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to SecretNode are documented here. This project adheres to ## [Unreleased] +### Added +- **Passive subdomain enumeration (deep-ASM slice 1).** New `backend/recon.py` expands a domain + into its known subdomain surface from **Certificate Transparency (crt.sh)** — fully passive, it + never contacts the target, so it runs before a client engagement is signed. `extract_registrable_ + domain()` normalises URL/host/IP inputs (with a two-label public-suffix table incl. `.bd`), and + enumeration fails closed (empty result + error string) on any network/parse error. Exposed via the + CLI: `python cli.py --subdomains`. First layer of the passive attack-surface pipeline + (subdomains → historical paths → associated assets → existing secret/posture scan). + ### Fixed - **False-negative: structural keys wrongly entropy-gated.** The Shannon-entropy floor (`MIN_ENTROPY_THRESHOLD=3.5`) was applied uniformly to every detector, silently dropping diff --git a/backend/cli.py b/backend/cli.py index ed2b500..13e26ad 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -27,6 +27,7 @@ import sys from urllib.parse import urlparse +import recon import report import scanner @@ -78,11 +79,41 @@ def build_parser() -> argparse.ArgumentParser: ap.add_argument("-o", "--output", help="Write report to this file (default: stdout)") ap.add_argument("--fail-on-findings", action="store_true", help="Exit non-zero if any confirmed findings (use as a CI gate)") + ap.add_argument("--subdomains", action="store_true", + help="Passive attack-surface discovery: enumerate the target's subdomains " + "via Certificate Transparency (crt.sh) and print them. Never contacts " + "the target. Authorized use only.") return ap +async def _run_subdomain_enum(target: str) -> int: + """Passive subdomain discovery mode: expand a domain into its known subdomain + surface from Certificate Transparency and print the results as JSON.""" + domain = recon.extract_registrable_domain(target) + if domain is None: + raise SystemExit( + f"Cannot enumerate subdomains for {target!r} — need a domain " + "(subdomain enumeration does not apply to bare IP addresses)." + ) + async with scanner.build_client() as client: + result = await recon.enumerate_subdomains_ct(client, domain) + print(json.dumps(result.to_dict(), indent=2)) + print( + f"SecretNode: discovered {result.count} subdomain(s) for {domain} " + f"(source: {result.source})." + (f" [error: {result.error}]" if result.error else ""), + file=sys.stderr, + ) + return 0 + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + + # Passive discovery mode: enumerate subdomains and exit. Accepts a bare domain + # (no scheme) since it never fetches the target — only Certificate Transparency. + if args.subdomains: + return asyncio.run(_run_subdomain_enum(args.target)) + if not args.target.startswith(("http://", "https://")): raise SystemExit("Target must start with http:// or https://") assert_public_target(args.target) diff --git a/backend/recon.py b/backend/recon.py new file mode 100644 index 0000000..dcfec9e --- /dev/null +++ b/backend/recon.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +""" +SecretNode passive reconnaissance — attack-surface discovery. + +The first discovery layer of the deep-ASM pipeline: given a domain, expand it +into its full known subdomain surface using **passive public data only**, so no +packet is ever sent to the target itself. This slice sources subdomains from +Certificate Transparency (crt.sh) — every publicly-trusted TLS certificate is +logged there, which makes CT the single richest passive source of a domain's +hostnames. + +Design rules (mirror the scanner's posture): + • Passive only — we query third-party CT logs, never the target. Nothing here + resolves DNS, connects to, or probes the target host. + • Fails closed — any network/parse error yields an empty result, never an + exception that would abort a scan. Discovery is best-effort by nature. + • Authorized use only — enumerating a domain's surface is reconnaissance; only + run it against assets you own or are explicitly permitted to assess. +""" + +from __future__ import annotations + +import ipaddress +import json +import logging +import os +from dataclasses import dataclass, field +from urllib.parse import urlparse + +import httpx + +logger = logging.getLogger("secretnode.recon") + + +def _env_int(name: str, default: int) -> int: + try: + return int(os.environ.get(name, "").strip() or default) + except ValueError: + return default + + +# crt.sh is the CT-log front-end; %25 is a URL-encoded '%' wildcard so the query +# "%.example.com" returns every logged hostname under the domain. +CRTSH_URL = os.environ.get("CRTSH_URL", "https://crt.sh").rstrip("/") +SUBDOMAIN_TIMEOUT = _env_int("SUBDOMAIN_ENUM_TIMEOUT", 30) +MAX_SUBDOMAINS = _env_int("MAX_SUBDOMAINS", 500) # safety cap on result size + +# A small public-suffix table so we can find the *registrable* domain of a host +# without a heavyweight PSL dependency. Covers the common two-label suffixes, +# with Bangladesh (Cindrasec's home market) explicitly included. Anything not +# listed falls back to the last two labels, which is correct for gTLDs. +_TWO_LEVEL_SUFFIXES: frozenset[str] = frozenset({ + "co.uk", "org.uk", "gov.uk", "ac.uk", "me.uk", "net.uk", "sch.uk", + "com.au", "net.au", "org.au", "edu.au", "gov.au", + "co.nz", "net.nz", "org.nz", "govt.nz", + "com.bd", "net.bd", "org.bd", "edu.bd", "gov.bd", "ac.bd", "mil.bd", + "co.in", "net.in", "org.in", "gen.in", "firm.in", "ind.in", + "com.br", "net.br", "org.br", "gov.br", + "com.sg", "com.my", "com.pk", "com.np", "com.lk", "com.cn", "com.hk", + "co.jp", "or.jp", "ne.jp", "co.kr", "co.za", "co.id", "co.th", +}) + + +@dataclass +class SubdomainResult: + """Outcome of a passive subdomain enumeration for one registrable domain.""" + domain: str + subdomains: list[str] = field(default_factory=list) + source: str = "crt.sh" + error: str | None = None + + @property + def count(self) -> int: + return len(self.subdomains) + + def to_dict(self) -> dict: + return { + "domain": self.domain, + "subdomains": self.subdomains, + "count": self.count, + "source": self.source, + "error": self.error, + } + + +def _host_of(target: str) -> str: + """Extract the bare hostname from a URL, host[:port], or bare domain.""" + target = (target or "").strip() + if not target: + return "" + if "://" in target: + target = urlparse(target).hostname or "" + else: + # Strip a trailing path and any :port, but keep bracketed IPv6 intact. + target = target.split("/", 1)[0] + if target.count(":") == 1: # host:port (not IPv6, which has many) + target = target.split(":", 1)[0] + return target.strip().strip(".").lower() + + +def is_ip_literal(target: str) -> bool: + """True if the target's host is an IP address (subdomain enum does not apply).""" + host = _host_of(target).strip("[]") + try: + ipaddress.ip_address(host) + return True + except ValueError: + return False + + +def extract_registrable_domain(target: str) -> str | None: + """Reduce a URL / host / bare domain to its registrable domain + (e.g. https://api.blog.example.co.uk/x -> example.co.uk). Returns None for IP + literals and inputs with no usable hostname, since neither can be enumerated.""" + host = _host_of(target) + if not host or is_ip_literal(target): + return None + labels = [x for x in host.split(".") if x] + if len(labels) < 2: + return None + last_two = ".".join(labels[-2:]) + if last_two in _TWO_LEVEL_SUFFIXES and len(labels) >= 3: + return ".".join(labels[-3:]) + return last_two + + +def _clean_name(name: str, domain: str) -> str | None: + """Normalise one CT 'name_value' entry into a single in-scope hostname, or + None if it is not a usable subdomain of `domain`.""" + n = (name or "").strip().lower().rstrip(".") + if n.startswith("*."): # wildcard cert — the base host is what matters + n = n[2:] + if not n or "@" in n or " " in n: # skip emails / malformed entries + return None + if n == domain or n.endswith("." + domain): + return n + return None + + +def parse_crtsh_json(payload: object, domain: str) -> list[str]: + """Parse a crt.sh JSON response into a sorted, deduplicated list of in-scope + hostnames. Pure function (no I/O) so it is cheap and deterministic to test. + + Accepts either an already-decoded list or a raw JSON string. crt.sh packs one + or more newline-separated names into each record's `name_value`, plus a + `common_name`; we harvest every name from both fields.""" + domain = (domain or "").strip().lower().rstrip(".") + if isinstance(payload, (str, bytes)): + try: + payload = json.loads(payload) + except (ValueError, TypeError): + return [] + if not isinstance(payload, list): + return [] + + found: set[str] = set() + for record in payload: + if not isinstance(record, dict): + continue + raw_names = str(record.get("name_value", "")).split("\n") + raw_names.append(str(record.get("common_name", ""))) + for raw in raw_names: + host = _clean_name(raw, domain) + if host: + found.add(host) + return sorted(found) + + +async def enumerate_subdomains_ct( + client: httpx.AsyncClient, + domain: str, + *, + limit: int = MAX_SUBDOMAINS, +) -> SubdomainResult: + """Passively enumerate subdomains of `domain` via Certificate Transparency. + + Never contacts the target — only crt.sh. Fails closed: on any error the + result carries an `error` string and an empty subdomain list so a caller can + proceed (and report the gap) instead of aborting.""" + domain = (domain or "").strip().lower().rstrip(".") + if not domain: + return SubdomainResult(domain=domain, error="empty domain") + + url = f"{CRTSH_URL}/?q=%25.{domain}&output=json" + try: + resp = await client.get(url, timeout=httpx.Timeout(SUBDOMAIN_TIMEOUT, connect=10.0)) + if resp.status_code != 200: + return SubdomainResult(domain=domain, error=f"crt.sh HTTP {resp.status_code}") + subs = parse_crtsh_json(resp.text, domain) + except (httpx.HTTPError, httpx.InvalidURL) as exc: + logger.debug("crt.sh enumeration failed for %s: %s", domain, exc) + return SubdomainResult(domain=domain, error=f"crt.sh request failed: {exc}") + + if len(subs) > limit: + logger.debug("crt.sh returned %d subdomains for %s; capping at %d", + len(subs), domain, limit) + subs = subs[:limit] + return SubdomainResult(domain=domain, subdomains=subs) diff --git a/backend/tests/test_recon.py b/backend/tests/test_recon.py new file mode 100644 index 0000000..e35ed43 --- /dev/null +++ b/backend/tests/test_recon.py @@ -0,0 +1,133 @@ +"""Tests for passive reconnaissance (subdomain enumeration via Certificate +Transparency). Network is mocked so these are fast and deterministic; the pure +parsing/normalisation helpers are exercised directly.""" + +from __future__ import annotations + +import json + +import pytest + +import recon + + +class TestExtractRegistrableDomain: + def test_bare_domain(self): + assert recon.extract_registrable_domain("example.com") == "example.com" + + def test_url_with_subdomain_and_path(self): + assert recon.extract_registrable_domain("https://api.blog.example.com/x?y=1") == "example.com" + + def test_host_with_port(self): + assert recon.extract_registrable_domain("shop.example.com:8443") == "example.com" + + def test_two_level_suffix_uk(self): + assert recon.extract_registrable_domain("https://api.example.co.uk/") == "example.co.uk" + + def test_two_level_suffix_bd(self): + # Cindrasec's home market — must not truncate to the public suffix itself. + assert recon.extract_registrable_domain("mail.gov.bd") == "mail.gov.bd" + assert recon.extract_registrable_domain("www.dhaka.gov.bd") == "dhaka.gov.bd" + + def test_ip_literal_returns_none(self): + assert recon.extract_registrable_domain("http://192.168.1.1/") is None + assert recon.extract_registrable_domain("10.0.0.5") is None + + def test_single_label_returns_none(self): + assert recon.extract_registrable_domain("localhost") is None + + +class TestIsIpLiteral: + def test_ipv4(self): + assert recon.is_ip_literal("https://203.0.113.5:8000/") is True + + def test_domain_is_not_ip(self): + assert recon.is_ip_literal("example.com") is False + + +class TestParseCrtshJson: + def test_harvests_and_dedupes_in_scope_names(self): + payload = [ + {"name_value": "api.example.com\nwww.example.com", "common_name": "example.com"}, + {"name_value": "www.example.com", "common_name": "cdn.example.com"}, + ] + out = recon.parse_crtsh_json(payload, "example.com") + assert out == ["api.example.com", "cdn.example.com", "example.com", "www.example.com"] + + def test_strips_wildcards(self): + payload = [{"name_value": "*.example.com", "common_name": ""}] + assert recon.parse_crtsh_json(payload, "example.com") == ["example.com"] + + def test_drops_out_of_scope_and_emails(self): + payload = [{ + "name_value": "good.example.com\nevil.example.com.attacker.net\nadmin@example.com", + "common_name": "notexample.com", + }] + # Only the genuine in-scope host survives; a look-alike suffix and an + # email address must both be rejected (no false surface). + assert recon.parse_crtsh_json(payload, "example.com") == ["good.example.com"] + + def test_accepts_raw_json_string(self): + payload = json.dumps([{"name_value": "a.example.com", "common_name": ""}]) + assert recon.parse_crtsh_json(payload, "example.com") == ["a.example.com"] + + def test_malformed_payload_yields_empty(self): + assert recon.parse_crtsh_json("not json", "example.com") == [] + assert recon.parse_crtsh_json({"unexpected": "shape"}, "example.com") == [] + + +class _FakeResponse: + def __init__(self, status_code: int, text: str): + self.status_code = status_code + self.text = text + + +class _FakeClient: + """Minimal async httpx.AsyncClient stand-in for enumerate_subdomains_ct.""" + def __init__(self, response=None, exc=None): + self._response = response + self._exc = exc + self.requested_url: str | None = None + + async def get(self, url, **kwargs): + self.requested_url = url + if self._exc is not None: + raise self._exc + return self._response + + +@pytest.mark.asyncio +async def test_enumerate_success(): + body = json.dumps([{"name_value": "api.example.com\nwww.example.com", "common_name": ""}]) + client = _FakeClient(response=_FakeResponse(200, body)) + result = await recon.enumerate_subdomains_ct(client, "example.com") + assert result.error is None + assert result.subdomains == ["api.example.com", "www.example.com"] + assert result.count == 2 + # Query must target crt.sh with the wildcard, never the target host itself. + assert "crt.sh" in client.requested_url and "%25.example.com" in client.requested_url + + +@pytest.mark.asyncio +async def test_enumerate_non_200_fails_closed(): + client = _FakeClient(response=_FakeResponse(503, "")) + result = await recon.enumerate_subdomains_ct(client, "example.com") + assert result.subdomains == [] + assert result.error is not None and "503" in result.error + + +@pytest.mark.asyncio +async def test_enumerate_network_error_fails_closed(): + import httpx + client = _FakeClient(exc=httpx.ConnectError("boom")) + result = await recon.enumerate_subdomains_ct(client, "example.com") + assert result.subdomains == [] + assert result.error is not None + + +@pytest.mark.asyncio +async def test_enumerate_respects_limit(): + names = "\n".join(f"h{i}.example.com" for i in range(10)) + client = _FakeClient(response=_FakeResponse(200, json.dumps([{"name_value": names}]))) + result = await recon.enumerate_subdomains_ct(client, "example.com", limit=3) + assert result.count == 3 From 75f9c49ad7b1e40bed64c83c8ebc9bfee3a0adf3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 18:58:54 +0000 Subject: [PATCH 05/19] Harden subdomain enumeration: multi-source + retries (crt.sh is flaky) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live Pi testing exposed crt.sh returning 502 / timing out — a single CT source is not production-grade, and the timeout surfaced as a blank error string. Make enumeration reliable the way subfinder/amass do: query several passive sources with retries and merge. • Second CT source: Certspotter (/v1/issuances), parsed by the new pure parse_certspotter_json(). Still passive — a third-party CT aggregator, never the target. • enumerate_subdomains() now queries crt.sh + Certspotter, unions and dedupes. `error` is set only if EVERY source fails, so one flaky source no longer zeroes out a good result; `sources` lists what worked. • _get_with_retries(): backoff retries on transient statuses (429/500/502/503/504) and timeouts, and every error names its type (fixes the blank "crt.sh request failed: " seen on the Pi). • SubdomainResult.source -> sources (list). CLI prints the source list. • Config: CERTSPOTTER_URL, SUBDOMAIN_ENUM_RETRIES. enumerate_subdomains_ct kept as a backward-compatible alias. Tests: Certspotter parsing, both-source merge, one-source-down still OK, retry-then-succeed, all-fail names each source, limit. Sleeps neutralised so retry paths run instantly. Suite 215 -> 219, ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- backend/cli.py | 5 +- backend/recon.py | 139 ++++++++++++++++++++++++++++++------ backend/tests/test_recon.py | 123 +++++++++++++++++++++++-------- 3 files changed, 216 insertions(+), 51 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index 13e26ad..ffe9b88 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -96,11 +96,12 @@ async def _run_subdomain_enum(target: str) -> int: "(subdomain enumeration does not apply to bare IP addresses)." ) async with scanner.build_client() as client: - result = await recon.enumerate_subdomains_ct(client, domain) + result = await recon.enumerate_subdomains(client, domain) print(json.dumps(result.to_dict(), indent=2)) + sources = ", ".join(result.sources) if result.sources else "none" print( f"SecretNode: discovered {result.count} subdomain(s) for {domain} " - f"(source: {result.source})." + (f" [error: {result.error}]" if result.error else ""), + f"(sources: {sources})." + (f" [error: {result.error}]" if result.error else ""), file=sys.stderr, ) return 0 diff --git a/backend/recon.py b/backend/recon.py index dcfec9e..a45fb60 100644 --- a/backend/recon.py +++ b/backend/recon.py @@ -20,6 +20,7 @@ from __future__ import annotations +import asyncio import ipaddress import json import logging @@ -42,8 +43,18 @@ def _env_int(name: str, default: int) -> int: # crt.sh is the CT-log front-end; %25 is a URL-encoded '%' wildcard so the query # "%.example.com" returns every logged hostname under the domain. CRTSH_URL = os.environ.get("CRTSH_URL", "https://crt.sh").rstrip("/") +# Certspotter is a second, independent CT aggregator. Querying more than one +# passive source is how industrial enumerators (subfinder/amass) stay reliable +# when any single source is rate-limited or down — crt.sh in particular returns +# 502/timeout often enough that a lone source is not production-grade. +CERTSPOTTER_URL = os.environ.get("CERTSPOTTER_URL", "https://api.certspotter.com").rstrip("/") SUBDOMAIN_TIMEOUT = _env_int("SUBDOMAIN_ENUM_TIMEOUT", 30) MAX_SUBDOMAINS = _env_int("MAX_SUBDOMAINS", 500) # safety cap on result size +ENUM_RETRIES = _env_int("SUBDOMAIN_ENUM_RETRIES", 2) # retries on transient CT errors + +# Transient HTTP statuses worth retrying — crt.sh/Certspotter throw these under +# load; a retry with backoff usually clears them. +_TRANSIENT_STATUS: frozenset[int] = frozenset({429, 500, 502, 503, 504}) # A small public-suffix table so we can find the *registrable* domain of a host # without a heavyweight PSL dependency. Covers the common two-label suffixes, @@ -63,10 +74,14 @@ def _env_int(name: str, default: int) -> int: @dataclass class SubdomainResult: - """Outcome of a passive subdomain enumeration for one registrable domain.""" + """Outcome of a passive subdomain enumeration for one registrable domain. + + `sources` lists the passive sources that actually returned data; `error` is + set only when *every* source failed (so a caller can distinguish "the domain + has no subdomains" from "enumeration could not run").""" domain: str subdomains: list[str] = field(default_factory=list) - source: str = "crt.sh" + sources: list[str] = field(default_factory=list) error: str | None = None @property @@ -78,7 +93,7 @@ def to_dict(self) -> dict: "domain": self.domain, "subdomains": self.subdomains, "count": self.count, - "source": self.source, + "sources": self.sources, "error": self.error, } @@ -166,33 +181,117 @@ def parse_crtsh_json(payload: object, domain: str) -> list[str]: return sorted(found) -async def enumerate_subdomains_ct( +def parse_certspotter_json(payload: object, domain: str) -> list[str]: + """Parse a Certspotter `/v1/issuances` response into sorted, in-scope + hostnames. Each issuance carries a `dns_names` list. Pure function.""" + domain = (domain or "").strip().lower().rstrip(".") + if isinstance(payload, (str, bytes)): + try: + payload = json.loads(payload) + except (ValueError, TypeError): + return [] + if not isinstance(payload, list): + return [] + + found: set[str] = set() + for record in payload: + if not isinstance(record, dict): + continue + for raw in record.get("dns_names", []) or []: + host = _clean_name(str(raw), domain) + if host: + found.add(host) + return sorted(found) + + +async def _get_with_retries( + client: httpx.AsyncClient, url: str, *, headers: dict | None = None, +) -> tuple[httpx.Response | None, str | None]: + """GET with backoff retries on transient statuses/timeouts. Returns + (response, None) on a final response, or (None, error) if every attempt + failed to produce one. Always names the failure type so errors are never + blank (the crt.sh timeout showed up as an empty string before).""" + last_err = "no attempt made" + for attempt in range(ENUM_RETRIES + 1): + try: + resp = await client.get( + url, headers=headers or {}, + timeout=httpx.Timeout(SUBDOMAIN_TIMEOUT, connect=10.0), + ) + if resp.status_code in _TRANSIENT_STATUS and attempt < ENUM_RETRIES: + last_err = f"HTTP {resp.status_code}" + await asyncio.sleep(2 ** attempt) + continue + return resp, None + except (httpx.HTTPError, httpx.InvalidURL) as exc: + last_err = f"{type(exc).__name__}: {exc}".rstrip(": ").strip() + if attempt < ENUM_RETRIES: + await asyncio.sleep(2 ** attempt) + continue + return None, last_err + + +async def _fetch_crtsh(client: httpx.AsyncClient, domain: str) -> tuple[set[str], str | None]: + resp, err = await _get_with_retries(client, f"{CRTSH_URL}/?q=%25.{domain}&output=json") + if resp is None: + return set(), err + if resp.status_code != 200: + return set(), f"HTTP {resp.status_code}" + return set(parse_crtsh_json(resp.text, domain)), None + + +async def _fetch_certspotter(client: httpx.AsyncClient, domain: str) -> tuple[set[str], str | None]: + url = (f"{CERTSPOTTER_URL}/v1/issuances?domain={domain}" + "&include_subdomains=true&expand=dns_names") + resp, err = await _get_with_retries(client, url) + if resp is None: + return set(), err + if resp.status_code != 200: + return set(), f"HTTP {resp.status_code}" + return set(parse_certspotter_json(resp.text, domain)), None + + +async def enumerate_subdomains( client: httpx.AsyncClient, domain: str, *, limit: int = MAX_SUBDOMAINS, ) -> SubdomainResult: - """Passively enumerate subdomains of `domain` via Certificate Transparency. + """Passively enumerate subdomains of `domain` from multiple Certificate + Transparency sources (crt.sh + Certspotter), merged and deduplicated. - Never contacts the target — only crt.sh. Fails closed: on any error the - result carries an `error` string and an empty subdomain list so a caller can - proceed (and report the gap) instead of aborting.""" + Never contacts the target. Fails closed: `error` is set only if *every* + source failed; if any source returns data the result is usable and lists the + sources that succeeded. Querying several sources is what keeps enumeration + reliable when one is rate-limited or down.""" domain = (domain or "").strip().lower().rstrip(".") if not domain: return SubdomainResult(domain=domain, error="empty domain") - url = f"{CRTSH_URL}/?q=%25.{domain}&output=json" - try: - resp = await client.get(url, timeout=httpx.Timeout(SUBDOMAIN_TIMEOUT, connect=10.0)) - if resp.status_code != 200: - return SubdomainResult(domain=domain, error=f"crt.sh HTTP {resp.status_code}") - subs = parse_crtsh_json(resp.text, domain) - except (httpx.HTTPError, httpx.InvalidURL) as exc: - logger.debug("crt.sh enumeration failed for %s: %s", domain, exc) - return SubdomainResult(domain=domain, error=f"crt.sh request failed: {exc}") - + all_hosts: set[str] = set() + ok_sources: list[str] = [] + errors: list[str] = [] + for name, fetch in (("crt.sh", _fetch_crtsh), ("certspotter", _fetch_certspotter)): + try: + hosts, err = await fetch(client, domain) + except Exception as exc: # defensive: a source must never crash the run + hosts, err = set(), f"{type(exc).__name__}: {exc}".rstrip(": ").strip() + if hosts: + all_hosts |= hosts + ok_sources.append(name) + if err: + errors.append(f"{name}: {err}") + + subs = sorted(all_hosts) if len(subs) > limit: - logger.debug("crt.sh returned %d subdomains for %s; capping at %d", + logger.debug("enumeration returned %d subdomains for %s; capping at %d", len(subs), domain, limit) subs = subs[:limit] - return SubdomainResult(domain=domain, subdomains=subs) + + error = None if ok_sources else ("; ".join(errors) or "no sources returned data") + return SubdomainResult(domain=domain, subdomains=subs, sources=ok_sources, error=error) + + +# Backward-compatible alias: the original single-source name now maps to the +# multi-source enumerator. +enumerate_subdomains_ct = enumerate_subdomains diff --git a/backend/tests/test_recon.py b/backend/tests/test_recon.py index e35ed43..05858ea 100644 --- a/backend/tests/test_recon.py +++ b/backend/tests/test_recon.py @@ -76,58 +76,123 @@ def test_malformed_payload_yields_empty(self): assert recon.parse_crtsh_json({"unexpected": "shape"}, "example.com") == [] -class _FakeResponse: - def __init__(self, status_code: int, text: str): +class TestParseCertspotterJson: + def test_harvests_dns_names(self): + payload = [ + {"dns_names": ["a.example.com", "*.example.com"]}, + {"dns_names": ["b.example.com", "admin@example.com"]}, + ] + # Wildcard collapses to the base host; the email is rejected. + assert recon.parse_certspotter_json(payload, "example.com") == [ + "a.example.com", "b.example.com", "example.com", + ] + + def test_malformed_yields_empty(self): + assert recon.parse_certspotter_json("nope", "example.com") == [] + + +class _Resp: + def __init__(self, status_code: int, text: str = ""): self.status_code = status_code self.text = text class _FakeClient: - """Minimal async httpx.AsyncClient stand-in for enumerate_subdomains_ct.""" - def __init__(self, response=None, exc=None): - self._response = response - self._exc = exc - self.requested_url: str | None = None + """Async httpx.AsyncClient stand-in that routes by URL substring. A route + value may be a response, an Exception (raised), or a list consumed in order + (last item repeats) — the list form lets us exercise retry-then-succeed.""" + def __init__(self, routes: dict): + self._routes = {k: (list(v) if isinstance(v, list) else v) for k, v in routes.items()} + self.requested_urls: list[str] = [] async def get(self, url, **kwargs): - self.requested_url = url - if self._exc is not None: - raise self._exc - return self._response + self.requested_urls.append(url) + val = next((v for k, v in self._routes.items() if k in url), _Resp(404)) + item = (val.pop(0) if len(val) > 1 else val[0]) if isinstance(val, list) else val + if isinstance(item, Exception): + raise item + return item + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + """Neutralise backoff sleeps so retry paths run instantly in tests.""" + async def _fast(*_a, **_k): + return None + monkeypatch.setattr(recon.asyncio, "sleep", _fast) + + +def _crtsh_body(*names: str) -> str: + return json.dumps([{"name_value": "\n".join(names), "common_name": ""}]) + + +def _certspotter_body(*names: str) -> str: + return json.dumps([{"dns_names": list(names)}]) @pytest.mark.asyncio -async def test_enumerate_success(): - body = json.dumps([{"name_value": "api.example.com\nwww.example.com", "common_name": ""}]) - client = _FakeClient(response=_FakeResponse(200, body)) - result = await recon.enumerate_subdomains_ct(client, "example.com") +async def test_enumerate_success_from_crtsh(): + client = _FakeClient({"crt.sh": _Resp(200, _crtsh_body("api.example.com", "www.example.com"))}) + result = await recon.enumerate_subdomains(client, "example.com") assert result.error is None assert result.subdomains == ["api.example.com", "www.example.com"] - assert result.count == 2 - # Query must target crt.sh with the wildcard, never the target host itself. - assert "crt.sh" in client.requested_url and "%25.example.com" in client.requested_url + assert "crt.sh" in result.sources + # crt.sh must be queried with the wildcard, never the target host itself. + assert any("crt.sh" in u and "%25.example.com" in u for u in client.requested_urls) @pytest.mark.asyncio -async def test_enumerate_non_200_fails_closed(): - client = _FakeClient(response=_FakeResponse(503, "")) - result = await recon.enumerate_subdomains_ct(client, "example.com") - assert result.subdomains == [] - assert result.error is not None and "503" in result.error +async def test_enumerate_merges_both_sources(): + client = _FakeClient({ + "crt.sh": _Resp(200, _crtsh_body("a.example.com")), + "certspotter": _Resp(200, _certspotter_body("b.example.com")), + }) + result = await recon.enumerate_subdomains(client, "example.com") + assert result.subdomains == ["a.example.com", "b.example.com"] + assert set(result.sources) == {"crt.sh", "certspotter"} + + +@pytest.mark.asyncio +async def test_enumerate_survives_one_source_down(): + client = _FakeClient({ + "crt.sh": _Resp(503), # down (after retries) + "certspotter": _Resp(200, _certspotter_body("b.example.com")), + }) + result = await recon.enumerate_subdomains(client, "example.com") + assert result.subdomains == ["b.example.com"] + assert result.sources == ["certspotter"] + assert result.error is None # at least one source worked → not an error + + +@pytest.mark.asyncio +async def test_enumerate_retries_transient_then_succeeds(): + # crt.sh returns 502 then 200 — the retry must recover it. + client = _FakeClient({ + "crt.sh": [_Resp(502), _Resp(200, _crtsh_body("a.example.com"))], + "certspotter": _Resp(429), + }) + result = await recon.enumerate_subdomains(client, "example.com") + assert result.subdomains == ["a.example.com"] + assert result.sources == ["crt.sh"] @pytest.mark.asyncio -async def test_enumerate_network_error_fails_closed(): +async def test_enumerate_all_sources_fail_sets_error(): import httpx - client = _FakeClient(exc=httpx.ConnectError("boom")) - result = await recon.enumerate_subdomains_ct(client, "example.com") + client = _FakeClient({ + "crt.sh": _Resp(503), + "certspotter": httpx.ConnectError("boom"), + }) + result = await recon.enumerate_subdomains(client, "example.com") assert result.subdomains == [] assert result.error is not None + # Error names each source's failure (never blank, unlike the old timeout bug). + assert "crt.sh" in result.error and "certspotter" in result.error @pytest.mark.asyncio async def test_enumerate_respects_limit(): - names = "\n".join(f"h{i}.example.com" for i in range(10)) - client = _FakeClient(response=_FakeResponse(200, json.dumps([{"name_value": names}]))) - result = await recon.enumerate_subdomains_ct(client, "example.com", limit=3) + names = [f"h{i}.example.com" for i in range(10)] + client = _FakeClient({"crt.sh": _Resp(200, _crtsh_body(*names))}) + result = await recon.enumerate_subdomains(client, "example.com", limit=3) assert result.count == 3 From 983a613e9c9db867032967d96052f4c5b370ea8b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 18:59:12 +0000 Subject: [PATCH 06/19] docs: changelog reflects multi-source subdomain enumeration Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- CHANGELOG.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6719fbe..a1c1a6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,13 @@ All notable changes to SecretNode are documented here. This project adheres to ### Added - **Passive subdomain enumeration (deep-ASM slice 1).** New `backend/recon.py` expands a domain - into its known subdomain surface from **Certificate Transparency (crt.sh)** — fully passive, it - never contacts the target, so it runs before a client engagement is signed. `extract_registrable_ - domain()` normalises URL/host/IP inputs (with a two-label public-suffix table incl. `.bd`), and - enumeration fails closed (empty result + error string) on any network/parse error. Exposed via the - CLI: `python cli.py --subdomains`. First layer of the passive attack-surface pipeline + into its known subdomain surface from **Certificate Transparency** — fully passive, it never + contacts the target, so it runs before a client engagement is signed. Queries **two independent CT + sources (crt.sh + Certspotter)** with backoff retries and merges them, so a single flaky/rate- + limited source (crt.sh 502s often) no longer zeroes out a good result; the result lists which + sources succeeded and only reports an error if *all* fail. `extract_registrable_domain()` + normalises URL/host/IP inputs (two-label public-suffix table incl. `.bd`). Exposed via the CLI: + `python cli.py --subdomains`. First layer of the passive attack-surface pipeline (subdomains → historical paths → associated assets → existing secret/posture scan). ### Fixed From 83e5de49e3f775df774c720d8249efbd68fa0605 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 19:08:00 +0000 Subject: [PATCH 07/19] Add multi-target orchestration: domain -> scan every live host (deep-ASM slice 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the discovery-to-findings loop. backend/orchestrator.py takes a single domain and: enumerates its subdomains (passive CT), probes which hosts are live, runs the existing passive secret+posture scan against each, and aggregates everything into one DeepScanResult / combined report. This is the "deepest scan, not just the surface" step — a surface scan of one host becomes a full-domain assessment. • run_deep_scan(): enumerate -> probe -> scan-each -> aggregate, with all collaborators injected (enumerate/scan/client) so it is unit-testable with no network. IP / non-enumerable targets degrade to a single scan. • SSRF guard: a discovered host that resolves to a private/internal address is skipped (recorded, not scanned) unless ALLOW_PRIVATE_TARGETS — a wildcard/misissued cert can name internal hosts, and the scanner itself has no built-in guard, so orchestration must enforce it. • Resilience: MAX_TARGETS cap, concurrent liveness probing (PROBE_CONCURRENCY), per-host error isolation (one host raising is recorded and the run continues). • report.generate_deep_scan_html(): one client deliverable across the domain — subdomain surface, live hosts, per-host confirmed/needs- review/posture with an EXPOSURE/CLEAN verdict. • CLI: `python cli.py --deep-scan -o report.html` (+ --max-targets). Config: MAX_TARGETS, PROBE_CONCURRENCY, PROBE_TIMEOUT. 10 orchestrator tests (probe filtering, aggregation, dead-host exclusion, max-targets, per-host error isolation, IP fallback, SSRF-guard skip); network fully mocked. Suite 219 -> 229, ruff clean. Passive posture intact. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- .env.example | 7 + CHANGELOG.md | 9 ++ backend/cli.py | 43 +++++ backend/orchestrator.py | 247 +++++++++++++++++++++++++++++ backend/report.py | 100 ++++++++++++ backend/tests/test_orchestrator.py | 185 +++++++++++++++++++++ 6 files changed, 591 insertions(+) create mode 100644 backend/orchestrator.py create mode 100644 backend/tests/test_orchestrator.py diff --git a/.env.example b/.env.example index c3dfac4..7598a71 100644 --- a/.env.example +++ b/.env.example @@ -73,6 +73,13 @@ CRTSH_URL=https://crt.sh SUBDOMAIN_ENUM_TIMEOUT=30 # Safety cap on how many discovered subdomains to return. MAX_SUBDOMAINS=500 +# ── Multi-target deep scan (domain → enumerate → probe → scan each host) ───── +# Max live hosts scanned in one --deep-scan run. +MAX_TARGETS=25 +# Parallel liveness probes when checking which discovered hosts are up. +PROBE_CONCURRENCY=10 +# Seconds per liveness probe before a host is treated as unreachable. +PROBE_TIMEOUT=10 # Skip JS assets larger than this many bytes (default 5 MB). MAX_ASSET_BYTES=5242880 # Only surface AI-confirmed findings at or above this confidence (0-100). diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c1a6f..dde7606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ All notable changes to SecretNode are documented here. This project adheres to ## [Unreleased] ### Added +- **Multi-target orchestration (deep-ASM slice 2).** New `backend/orchestrator.py` closes the loop + from discovery to findings: a single domain → passive subdomain enumeration → liveness probe of + each host → the existing passive secret+posture scan per live host → one aggregated + `DeepScanResult`. Includes a per-host **SSRF guard** (a discovered host that resolves to a + private/internal address is skipped unless `ALLOW_PRIVATE_TARGETS=true`), a `MAX_TARGETS` cap, + concurrent probing, and per-host error isolation (one host failing never sinks the run). New + combined client report `report.generate_deep_scan_html()` (subdomain surface + live hosts + + per-host confirmed/needs-review/posture). CLI: `python cli.py --deep-scan -o report.html`. + Config: `MAX_TARGETS`, `PROBE_CONCURRENCY`, `PROBE_TIMEOUT`. - **Passive subdomain enumeration (deep-ASM slice 1).** New `backend/recon.py` expands a domain into its known subdomain surface from **Certificate Transparency** — fully passive, it never contacts the target, so it runs before a client engagement is signed. Queries **two independent CT diff --git a/backend/cli.py b/backend/cli.py index ffe9b88..03f5d40 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -27,6 +27,7 @@ import sys from urllib.parse import urlparse +import orchestrator import recon import report import scanner @@ -83,9 +84,46 @@ def build_parser() -> argparse.ArgumentParser: help="Passive attack-surface discovery: enumerate the target's subdomains " "via Certificate Transparency (crt.sh) and print them. Never contacts " "the target. Authorized use only.") + ap.add_argument("--deep-scan", dest="deep_scan", action="store_true", + help="Domain-wide deep scan: enumerate subdomains, probe which hosts are live, " + "scan each, and write a combined report. Passive; authorized use only.") + ap.add_argument("--max-targets", dest="max_targets", type=int, default=orchestrator.MAX_TARGETS, + help=f"Max live hosts to scan in a --deep-scan run (default: {orchestrator.MAX_TARGETS})") return ap +async def _run_deep_scan(args) -> int: + """Domain-wide deep scan: enumerate → probe live hosts → scan each → combined report.""" + result = await orchestrator.run_deep_scan( + args.target, + max_crawl_pages=max(1, args.crawl), + verify=args.verify, + only_verified=args.only_verified, + max_targets=max(1, args.max_targets), + ) + if args.output: + report_fmt = (args.format if args.format in ("json",) else "html") + body = (json.dumps(result.to_dict(), indent=2) if report_fmt == "json" + else report.generate_deep_scan_html(result.to_dict())) + with open(args.output, "w", encoding="utf-8") as fh: + fh.write(body) + print(f"Deep-scan report written to {args.output}", file=sys.stderr) + else: + sys.stdout.write(json.dumps(result.to_dict(), indent=2) + "\n") + + t = result.to_dict()["totals"] + print( + f"SecretNode deep scan of {result.domain}: {t['subdomains']} subdomain(s), " + f"{t['live_hosts']} live, {t['hosts_scanned']} scanned — " + f"{t['confirmed']} confirmed, {t['needs_review']} needs-review, " + f"{t['posture_issues']} posture issue(s).", + file=sys.stderr, + ) + if args.fail_on_findings and t["confirmed"]: + return 1 + return 0 + + async def _run_subdomain_enum(target: str) -> int: """Passive subdomain discovery mode: expand a domain into its known subdomain surface from Certificate Transparency and print the results as JSON.""" @@ -115,6 +153,11 @@ def main(argv: list[str] | None = None) -> int: if args.subdomains: return asyncio.run(_run_subdomain_enum(args.target)) + # Domain-wide deep scan: enumerate → probe → scan each live host. Accepts a + # bare domain; the orchestrator applies the SSRF guard per discovered host. + if args.deep_scan: + return asyncio.run(_run_deep_scan(args)) + if not args.target.startswith(("http://", "https://")): raise SystemExit("Target must start with http:// or https://") assert_public_target(args.target) diff --git a/backend/orchestrator.py b/backend/orchestrator.py new file mode 100644 index 0000000..f6648be --- /dev/null +++ b/backend/orchestrator.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +SecretNode multi-target orchestration — deep-ASM slice 2. + +Closes the loop from discovery to findings: given a single domain, expand it to +its subdomain surface (passive CT enumeration), probe which of those hosts are +actually live, then run the existing passive secret+posture scan against each +one and aggregate the results into a single deliverable. + +Everything here stays within SecretNode's passive/authorized posture: + • Enumeration is passive (Certificate Transparency, never the target). + • Liveness probing and scanning are the same passive fetches the single-target + scanner already performs — no exploitation, no brute-force, read-only. + • Authorized use only. Scanning a whole domain's host list at once magnifies + the responsibility: run it only against assets you own or are explicitly + permitted to assess (owned / signed RoE / in-scope program). + +Collaborators (enumeration, client factory, per-host scan) are injected so the +orchestration is unit-testable without any network. +""" + +from __future__ import annotations + +import asyncio +import ipaddress +import logging +import os +import socket +from dataclasses import dataclass, field +from typing import Awaitable, Callable + +import httpx + +import recon +import scanner + +logger = logging.getLogger("secretnode.orchestrator") + + +def _env_int(name: str, default: int) -> int: + try: + return int(os.environ.get(name, "").strip() or default) + except ValueError: + return default + + +MAX_TARGETS = _env_int("MAX_TARGETS", 25) # cap hosts scanned per run +PROBE_CONCURRENCY = _env_int("PROBE_CONCURRENCY", 10) # parallel liveness probes +PROBE_TIMEOUT = _env_int("PROBE_TIMEOUT", 10) # seconds per liveness probe + +# Type aliases for the injected collaborators. +EnumerateFn = Callable[..., Awaitable["recon.SubdomainResult"]] +ScanFn = Callable[..., Awaitable[dict]] +ClientFactory = Callable[[], httpx.AsyncClient] + + +@dataclass +class HostScan: + """Per-host outcome inside a deep scan.""" + host: str + url: str + confirmed: int = 0 + needs_review: int = 0 + posture_issues: int = 0 + assets: int = 0 + error: str | None = None + + def to_dict(self) -> dict: + return { + "host": self.host, "url": self.url, "confirmed": self.confirmed, + "needs_review": self.needs_review, "posture_issues": self.posture_issues, + "assets": self.assets, "error": self.error, + } + + +@dataclass +class DeepScanResult: + """Aggregate of a domain-wide deep scan.""" + domain: str + subdomains: list[str] = field(default_factory=list) + enum_sources: list[str] = field(default_factory=list) + live_hosts: list[str] = field(default_factory=list) + hosts: list[HostScan] = field(default_factory=list) + scans: list[dict] = field(default_factory=list) # raw per-host scan dicts + error: str | None = None + + @property + def total_confirmed(self) -> int: + return sum(h.confirmed for h in self.hosts) + + @property + def total_needs_review(self) -> int: + return sum(h.needs_review for h in self.hosts) + + @property + def total_posture(self) -> int: + return sum(h.posture_issues for h in self.hosts) + + def to_dict(self) -> dict: + return { + "domain": self.domain, + "subdomains": self.subdomains, + "enum_sources": self.enum_sources, + "live_hosts": self.live_hosts, + "hosts": [h.to_dict() for h in self.hosts], + "totals": { + "subdomains": len(self.subdomains), + "live_hosts": len(self.live_hosts), + "hosts_scanned": len(self.hosts), + "confirmed": self.total_confirmed, + "needs_review": self.total_needs_review, + "posture_issues": self.total_posture, + }, + "error": self.error, + } + + +def _target_ip_class(host: str) -> str: + """Classify a host by the address it resolves to: 'public', 'private' + (loopback/private/link-local/reserved/multicast — an SSRF risk), or + 'unresolved'. Used to keep a discovered host list from ever pointing the + scanner at internal infrastructure.""" + try: + infos = socket.getaddrinfo(host, None) + except socket.gaierror: + return "unresolved" + for info in infos: + ip = ipaddress.ip_address(info[4][0]) + if (ip.is_private or ip.is_loopback or ip.is_link_local + or ip.is_reserved or ip.is_multicast): + return "private" + return "public" + + +async def _probe_one(client: httpx.AsyncClient, host: str) -> str | None: + """Return the reachable base URL for a host (preferring https), or None if it + is unreachable. ANY HTTP response — including 401/403/5xx — means the host is + live and worth scanning; only a transport error (DNS/connect/timeout) is dead.""" + for scheme in ("https", "http"): + url = f"{scheme}://{host}" + try: + await client.get(url, timeout=httpx.Timeout(PROBE_TIMEOUT, connect=10.0)) + return url + except httpx.HTTPError: + continue + return None + + +async def probe_live_hosts( + client: httpx.AsyncClient, hosts: list[str], *, concurrency: int = PROBE_CONCURRENCY, +) -> list[str]: + """Concurrently probe hosts; return the reachable base URLs, order preserved.""" + sem = asyncio.Semaphore(max(1, concurrency)) + + async def _guarded(h: str) -> str | None: + async with sem: + return await _probe_one(client, h) + + results = await asyncio.gather(*(_guarded(h) for h in hosts)) + return [u for u in results if u] + + +def _summarise_scan(host: str, url: str, scan: dict) -> HostScan: + return HostScan( + host=host, + url=url, + confirmed=len(scan.get("confirmed_findings", [])), + needs_review=len(scan.get("needs_review_findings", [])), + posture_issues=len(scan.get("posture_findings", [])), + assets=int(scan.get("assets_fetched", 0) or 0), + error=scan.get("error"), + ) + + +async def run_deep_scan( + target: str, + *, + max_crawl_pages: int = 1, + verify: bool = False, + only_verified: bool = False, + max_targets: int = MAX_TARGETS, + enumerate_fn: EnumerateFn = recon.enumerate_subdomains, + scan_fn: ScanFn = scanner.run_scan, + client_factory: ClientFactory = scanner.build_client, +) -> DeepScanResult: + """Domain → enumerate → probe → scan each live host → aggregate. + + Falls back to scanning the bare target itself when the input is an IP or has + no enumerable domain, so a deep scan always does *something* useful.""" + domain = recon.extract_registrable_domain(target) + if domain is None: + # No enumerable domain (e.g. an IP): degrade to a single passive scan of + # the given target so the deep-scan entry point is still usable. + host = recon._host_of(target) or target + url = target if "://" in target else f"https://{host}" + result = DeepScanResult(domain=host) + scan = await scan_fn(target_url=url, max_crawl_pages=max_crawl_pages, + verify=verify, only_verified=only_verified) + result.live_hosts = [url] + result.hosts = [_summarise_scan(host, url, scan)] + result.scans = [scan] + return result + + result = DeepScanResult(domain=domain) + async with client_factory() as client: + enum = await enumerate_fn(client, domain) + result.subdomains = enum.subdomains + result.enum_sources = enum.sources + # Always include the apex domain as a candidate even if CT omitted it. + candidates = list(dict.fromkeys([domain, *enum.subdomains])) + + # SSRF guard: never probe/scan a discovered host that resolves to an + # internal address (a wildcard/misissued cert can name one). Bypassed + # only by the same ALLOW_PRIVATE_TARGETS opt-in the single-target path uses. + allow_private = os.environ.get("ALLOW_PRIVATE_TARGETS", "false").lower() == "true" + safe_hosts: list[str] = [] + for host in candidates: + cls = "public" if allow_private else await asyncio.to_thread(_target_ip_class, host) + if cls == "public": + safe_hosts.append(host) + elif cls == "private": + result.hosts.append(HostScan( + host=host, url=f"https://{host}", + error="skipped: resolves to a private/internal address (SSRF guard)")) + # 'unresolved' hosts are silently dropped — nothing to scan. + + result.live_hosts = await probe_live_hosts(client, safe_hosts) + + if not result.live_hosts: + result.error = enum.error or "no live hosts found" + return result + + targets = result.live_hosts[:max(1, max_targets)] + for url in targets: + host = recon._host_of(url) + try: + scan = await scan_fn(target_url=url, max_crawl_pages=max_crawl_pages, + verify=verify, only_verified=only_verified) + except Exception as exc: # a single host must not sink the whole run + logger.debug("deep scan: host %s failed: %s", url, exc) + result.hosts.append(HostScan(host=host, url=url, + error=f"{type(exc).__name__}: {exc}".strip(": "))) + continue + result.scans.append(scan) + result.hosts.append(_summarise_scan(host, url, scan)) + + return result diff --git a/backend/report.py b/backend/report.py index 6b5deb7..966c7a7 100644 --- a/backend/report.py +++ b/backend/report.py @@ -537,3 +537,103 @@ def generate_sarif_report(scan: dict[str, Any]) -> str: }], } return json.dumps(sarif, indent=2) + + +# ───────────────────────────────────────────────────────────────────────────── +# Deep-scan (multi-target) summary report — deep-ASM slice 2 +# ───────────────────────────────────────────────────────────────────────────── + +def generate_deep_scan_html(deep: dict[str, Any]) -> str: + """Render a single client-facing summary for a domain-wide deep scan: the + discovered subdomain surface, which hosts were live, and per-host findings. + Input is orchestrator.DeepScanResult.to_dict().""" + domain = html.escape(str(deep.get("domain", ""))) + totals = deep.get("totals", {}) + hosts = deep.get("hosts", []) + sources = ", ".join(deep.get("enum_sources", [])) or "—" + generated = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + confirmed_total = int(totals.get("confirmed", 0)) + pill_color = "#c53030" if confirmed_total else "#276749" + pill_text = (f"{confirmed_total} confirmed credential exposure(s) across the domain" + if confirmed_total else "No confirmed credential exposures across the domain") + + def host_row(h: dict[str, Any]) -> str: + err = h.get("error") + conf = int(h.get("confirmed", 0)) + conf_cell = (f'{conf}' if conf else "0") + status = ('error' if err else "scanned") + note = html.escape(str(err)) if err else "" + return ( + "" + f'' + f'' + f'' + f'' + f'' + f'' + f'' + "" + ) + + rows = "\n".join(host_row(h) for h in hosts) or \ + '' + subs = ", ".join(html.escape(s) for s in deep.get("subdomains", [])) or "—" + + return f""" + +Domain Attack-Surface Report — {domain} + +

Domain Attack-Surface Report — {domain}

+
{'EXPOSURE' if confirmed_total else 'CLEAN'} +  {pill_text}. +
Passive assessment — subdomains discovered from + Certificate Transparency ({html.escape(sources)}); live hosts scanned for exposed credentials and + security-header posture. No exploitation, brute-force, or write operations were performed.
+
+ +
+
{int(totals.get('subdomains', 0))}
Subdomains found
+
{int(totals.get('live_hosts', 0))}
Live hosts
+
{int(totals.get('hosts_scanned', 0))}
Hosts scanned
+
{confirmed_total}
Confirmed exposures
+
{int(totals.get('needs_review', 0))}
Needs review
+
{int(totals.get('posture_issues', 0))}
Posture issues
+
+ +

Per-host results

+
SeverityTypeLocationNote
{html.escape(str(h.get("host", "")))}{status}{int(h.get("assets", 0))}{conf_cell}{int(h.get("needs_review", 0))}{int(h.get("posture_issues", 0))}{note}
No live hosts were scanned.
+ + {rows} +
HostStatusAssetsConfirmedNeeds reviewPostureNote
+ +

Discovered subdomain surface

+
{subs}
+ +
Generated by SecretNode v{_TOOL_VERSION} — passive attack-surface scanner. + Report generated {generated}. Discovery via Certificate Transparency; all host scans passive + (no exploitation, data exfiltration, or write operations). Authorized-scope testing only.
+""" diff --git a/backend/tests/test_orchestrator.py b/backend/tests/test_orchestrator.py new file mode 100644 index 0000000..4335358 --- /dev/null +++ b/backend/tests/test_orchestrator.py @@ -0,0 +1,185 @@ +"""Tests for multi-target orchestration (deep-ASM slice 2). Enumeration, the HTTP +client, and per-host scans are all injected/mocked, so these run offline and +deterministically.""" + +from __future__ import annotations + +import httpx +import pytest + +import orchestrator +import recon + + +class _Resp: + def __init__(self, status_code: int = 200): + self.status_code = status_code + + +class _FakeClient: + """Async client stand-in: probes succeed unless the host matches `dead`.""" + def __init__(self, dead: tuple[str, ...] = ()): + self.dead = set(dead) + self.gets: list[str] = [] + + async def get(self, url, **_kw): + self.gets.append(url) + if any(d in url for d in self.dead): + raise httpx.ConnectError("dead host") + return _Resp(200) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_a): + return False + + +def _client_factory(dead: tuple[str, ...] = ()): + def _factory(): + return _FakeClient(dead) + return _factory + + +def _enum_returning(subs, sources=("crt.sh",), error=None): + async def _enum(_client, domain): + return recon.SubdomainResult(domain=domain, subdomains=list(subs), + sources=list(sources), error=error) + return _enum + + +def _scan_returning(confirmed=0, needs_review=0, posture=0, assets=5): + async def _scan(*, target_url, **_kw): + return { + "target_url": target_url, + "confirmed_findings": [{"x": i} for i in range(confirmed)], + "needs_review_findings": [{"x": i} for i in range(needs_review)], + "posture_findings": [{"x": i} for i in range(posture)], + "assets_fetched": assets, + } + return _scan + + +@pytest.fixture(autouse=True) +def _allow_private(monkeypatch): + """Bypass the DNS SSRF guard by default so fake hostnames aren't dropped; + the guard itself is tested explicitly below.""" + monkeypatch.setenv("ALLOW_PRIVATE_TARGETS", "true") + + +class TestTargetIpClass: + def test_loopback_is_private(self): + assert orchestrator._target_ip_class("127.0.0.1") == "private" + + def test_public_ip_is_public(self): + assert orchestrator._target_ip_class("93.184.216.34") == "public" + + def test_unresolvable_is_unresolved(self): + # .invalid is reserved to never resolve (RFC 2606) — no network needed. + assert orchestrator._target_ip_class("nothing.invalid") == "unresolved" + + +@pytest.mark.asyncio +async def test_probe_live_hosts_filters_dead(): + client = _FakeClient(dead=("b.example.com",)) + live = await orchestrator.probe_live_hosts(client, ["a.example.com", "b.example.com"]) + assert live == ["https://a.example.com"] + + +@pytest.mark.asyncio +async def test_deep_scan_enumerates_probes_and_aggregates(): + result = await orchestrator.run_deep_scan( + "example.com", + enumerate_fn=_enum_returning(["a.example.com", "b.example.com"]), + scan_fn=_scan_returning(confirmed=1, posture=2), + client_factory=_client_factory(), + ) + d = result.to_dict() + # apex + 2 subdomains, all live, all scanned. + assert d["totals"]["subdomains"] == 2 + assert d["totals"]["live_hosts"] == 3 + assert d["totals"]["hosts_scanned"] == 3 + assert d["totals"]["confirmed"] == 3 # 1 per host × 3 + assert d["totals"]["posture_issues"] == 6 # 2 per host × 3 + assert "example.com" in result.subdomains or result.domain == "example.com" + + +@pytest.mark.asyncio +async def test_deep_scan_dead_host_excluded(): + result = await orchestrator.run_deep_scan( + "example.com", + enumerate_fn=_enum_returning(["a.example.com", "dead.example.com"]), + scan_fn=_scan_returning(confirmed=1), + client_factory=_client_factory(dead=("dead.example.com",)), + ) + hosts = [h.host for h in result.hosts] + assert "dead.example.com" not in hosts + assert result.to_dict()["totals"]["hosts_scanned"] == 2 # apex + a + + +@pytest.mark.asyncio +async def test_deep_scan_respects_max_targets(): + result = await orchestrator.run_deep_scan( + "example.com", + enumerate_fn=_enum_returning(["a.example.com", "b.example.com", "c.example.com"]), + scan_fn=_scan_returning(confirmed=1), + client_factory=_client_factory(), + max_targets=1, + ) + assert result.to_dict()["totals"]["hosts_scanned"] == 1 + + +@pytest.mark.asyncio +async def test_deep_scan_one_host_scan_error_is_isolated(): + calls = {"n": 0} + + async def _flaky_scan(*, target_url, **_kw): + calls["n"] += 1 + if "a.example.com" in target_url: + raise RuntimeError("boom") + return {"target_url": target_url, "confirmed_findings": [], + "needs_review_findings": [], "posture_findings": [], "assets_fetched": 1} + + result = await orchestrator.run_deep_scan( + "example.com", + enumerate_fn=_enum_returning(["a.example.com"]), + scan_fn=_flaky_scan, + client_factory=_client_factory(), + ) + errored = [h for h in result.hosts if h.error] + assert any("boom" in (h.error or "") for h in errored) + # The other host still scanned — one failure does not sink the run. + assert any(h.error is None for h in result.hosts) + + +@pytest.mark.asyncio +async def test_deep_scan_ip_target_falls_back_to_single_scan(): + result = await orchestrator.run_deep_scan( + "http://93.184.216.34", + enumerate_fn=_enum_returning(["should.not.be.used"]), + scan_fn=_scan_returning(confirmed=1), + client_factory=_client_factory(), + ) + # IP has no enumerable domain → exactly one host scanned (the target itself). + assert result.to_dict()["totals"]["hosts_scanned"] == 1 + assert result.hosts[0].url == "http://93.184.216.34" + + +@pytest.mark.asyncio +async def test_deep_scan_ssrf_guard_skips_private_host(monkeypatch): + # Turn the guard back on and force one host to look internal. + monkeypatch.setenv("ALLOW_PRIVATE_TARGETS", "false") + monkeypatch.setattr( + orchestrator, "_target_ip_class", + lambda host: "private" if host == "internal.example.com" else "public", + ) + result = await orchestrator.run_deep_scan( + "example.com", + enumerate_fn=_enum_returning(["internal.example.com", "a.example.com"]), + scan_fn=_scan_returning(confirmed=1), + client_factory=_client_factory(), + ) + skipped = [h for h in result.hosts if h.error and "SSRF" in h.error] + assert any(h.host == "internal.example.com" for h in skipped) + # The private host must never appear among the live/scanned hosts. + assert all("internal.example.com" not in u for u in result.live_hosts) From e7882057646ab0410ca06a62c61017f602612e00 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 19:18:45 +0000 Subject: [PATCH 08/19] Add historical path discovery from public archives (deep-ASM slice 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The passive alternative to directory/content brute-forcing: instead of guessing paths against the target (active, noisy, needs an RoE), recover the URLs it has already exposed from public web archives — no request touches the target. backend/historical.py: • Two sources merged: Wayback Machine CDX (matchType=domain, so apex + every subdomain) and CommonCrawl (two-hop: newest index via collinfo, then its CDX API). Same multi-source + backoff-retry + fail-closed design as the subdomain layer; error is set only if all sources fail. • Pure parsers parse_wayback_cdx() / parse_commoncrawl_jsonl() with in-scope host filtering (host == domain or *.domain). • HistoricalResult exposes urls, a unique-path view (the "hidden directories"), and js_urls() — the highest-value scan seeds, since bundled JS is where secrets leak most. • CLI: `python cli.py --historical`. ENABLE_COMMONCRAWL toggle + timeout/retry/limit config. Surfaces forgotten endpoints and stale bundles a live crawl never links to, and gives the testphp-style empty-crawl case real coverage. 10 new tests (parsers, merge, one-source-down, retry, all-fail, limit); network mocked. Suite 229 -> 239, ruff clean. Passive posture intact. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- .env.example | 10 ++ CHANGELOG.md | 9 ++ backend/cli.py | 31 ++++ backend/historical.py | 247 +++++++++++++++++++++++++++++++ backend/tests/test_historical.py | 153 +++++++++++++++++++ 5 files changed, 450 insertions(+) create mode 100644 backend/historical.py create mode 100644 backend/tests/test_historical.py diff --git a/.env.example b/.env.example index 7598a71..7f7bf26 100644 --- a/.env.example +++ b/.env.example @@ -80,6 +80,16 @@ MAX_TARGETS=25 PROBE_CONCURRENCY=10 # Seconds per liveness probe before a host is treated as unreachable. PROBE_TIMEOUT=10 +# ── Historical path discovery (passive, from public web archives) ─────────── +# Recover historically-exposed URLs (the passive alternative to brute-forcing). +WAYBACK_CDX_URL=http://web.archive.org/cdx/search/cdx +COMMONCRAWL_COLLINFO=https://index.commoncrawl.org/collinfo.json +# Set false to skip CommonCrawl and use the Wayback Machine only (faster). +ENABLE_COMMONCRAWL=true +HISTORICAL_TIMEOUT=30 +HISTORICAL_RETRIES=2 +# Safety cap on how many historical URLs to return. +MAX_HISTORICAL_URLS=2000 # Skip JS assets larger than this many bytes (default 5 MB). MAX_ASSET_BYTES=5242880 # Only surface AI-confirmed findings at or above this confidence (0-100). diff --git a/CHANGELOG.md b/CHANGELOG.md index dde7606..3008b4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ All notable changes to SecretNode are documented here. This project adheres to ## [Unreleased] ### Added +- **Historical path discovery (deep-ASM slice 3).** New `backend/historical.py` recovers a domain's + historically-exposed URLs from **public web archives (Wayback Machine + CommonCrawl)** — the + passive alternative to directory/content brute-forcing, so no request ever touches the target. Two + sources merged with backoff retries and fail-closed handling (matching the subdomain layer); + surfaces forgotten endpoints, stale JS bundles and old admin paths a live crawl would never link + to. `HistoricalResult` exposes the raw URLs, the unique-path view ("hidden directories"), and a + `js_urls()` helper (highest-value scan seeds). CLI: `python cli.py --historical`. Config: + `WAYBACK_CDX_URL`, `COMMONCRAWL_COLLINFO`, `ENABLE_COMMONCRAWL`, `HISTORICAL_TIMEOUT`, + `HISTORICAL_RETRIES`, `MAX_HISTORICAL_URLS`. - **Multi-target orchestration (deep-ASM slice 2).** New `backend/orchestrator.py` closes the loop from discovery to findings: a single domain → passive subdomain enumeration → liveness probe of each host → the existing passive secret+posture scan per live host → one aggregated diff --git a/backend/cli.py b/backend/cli.py index 03f5d40..5ad1f09 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -27,6 +27,7 @@ import sys from urllib.parse import urlparse +import historical import orchestrator import recon import report @@ -84,6 +85,10 @@ def build_parser() -> argparse.ArgumentParser: help="Passive attack-surface discovery: enumerate the target's subdomains " "via Certificate Transparency (crt.sh) and print them. Never contacts " "the target. Authorized use only.") + ap.add_argument("--historical", action="store_true", + help="Passive path discovery: recover historically-exposed URLs for the domain " + "from public web archives (Wayback Machine + CommonCrawl) and print them. " + "Never contacts the target. Authorized use only.") ap.add_argument("--deep-scan", dest="deep_scan", action="store_true", help="Domain-wide deep scan: enumerate subdomains, probe which hosts are live, " "scan each, and write a combined report. Passive; authorized use only.") @@ -145,6 +150,28 @@ async def _run_subdomain_enum(target: str) -> int: return 0 +async def _run_historical(target: str) -> int: + """Passive path discovery: recover historically-exposed URLs from public + archives (Wayback + CommonCrawl) — the passive alternative to brute-forcing.""" + domain = recon.extract_registrable_domain(target) + if domain is None: + raise SystemExit( + f"Cannot run historical discovery for {target!r} — need a domain " + "(does not apply to bare IP addresses)." + ) + async with scanner.build_client() as client: + result = await historical.discover_historical_urls(client, domain) + print(json.dumps(result.to_dict(), indent=2)) + sources = ", ".join(result.sources) if result.sources else "none" + print( + f"SecretNode: recovered {result.count} historical URL(s) " + f"({len(result.paths)} unique path(s), {len(result.js_urls())} JS) for {domain} " + f"(sources: {sources})." + (f" [error: {result.error}]" if result.error else ""), + file=sys.stderr, + ) + return 0 + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) @@ -153,6 +180,10 @@ def main(argv: list[str] | None = None) -> int: if args.subdomains: return asyncio.run(_run_subdomain_enum(args.target)) + # Passive historical path discovery from public archives; accepts a bare domain. + if args.historical: + return asyncio.run(_run_historical(args.target)) + # Domain-wide deep scan: enumerate → probe → scan each live host. Accepts a # bare domain; the orchestrator applies the SSRF guard per discovered host. if args.deep_scan: diff --git a/backend/historical.py b/backend/historical.py new file mode 100644 index 0000000..93da67a --- /dev/null +++ b/backend/historical.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +SecretNode historical path discovery — deep-ASM slice 3. + +The passive answer to directory/content brute-forcing. Instead of hammering a +target with guessed paths (active, noisy, needs a signed RoE), we recover the +URLs it has *already exposed* from public web archives: + + • the Wayback Machine (Internet Archive) CDX index, and + • CommonCrawl's URL index. + +Both are third-party archives — no request is ever sent to the target. This +surfaces forgotten endpoints, old admin panels, stale JS bundles and API paths +that a live crawl of the current site would never link to, which is exactly +where credentials tend to linger. + +Posture rules mirror the rest of the scanner: passive only, fails closed (any +source erroring yields an empty contribution, never an exception), and +authorized-scope use only. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from dataclasses import dataclass, field +from urllib.parse import urlparse + +import httpx + +logger = logging.getLogger("secretnode.historical") + + +def _env_int(name: str, default: int) -> int: + try: + return int(os.environ.get(name, "").strip() or default) + except ValueError: + return default + + +WAYBACK_CDX_URL = os.environ.get("WAYBACK_CDX_URL", "http://web.archive.org/cdx/search/cdx") +COMMONCRAWL_COLLINFO = os.environ.get("COMMONCRAWL_COLLINFO", "https://index.commoncrawl.org/collinfo.json") +HISTORICAL_TIMEOUT = _env_int("HISTORICAL_TIMEOUT", 30) +HISTORICAL_RETRIES = _env_int("HISTORICAL_RETRIES", 2) +MAX_HISTORICAL_URLS = _env_int("MAX_HISTORICAL_URLS", 2000) +ENABLE_COMMONCRAWL = os.environ.get("ENABLE_COMMONCRAWL", "true").lower() == "true" + +_TRANSIENT_STATUS: frozenset[int] = frozenset({429, 500, 502, 503, 504}) + + +@dataclass +class HistoricalResult: + """Historical URLs recovered for a domain from public archives.""" + domain: str + urls: list[str] = field(default_factory=list) + sources: list[str] = field(default_factory=list) + error: str | None = None + + @property + def count(self) -> int: + return len(self.urls) + + @property + def paths(self) -> list[str]: + """Unique URL paths across all discovered URLs — the 'hidden directories' + view (each distinct path an archive has seen for this domain).""" + seen: set[str] = set() + for u in self.urls: + p = urlparse(u).path or "/" + seen.add(p) + return sorted(seen) + + def js_urls(self) -> list[str]: + """Discovered URLs that look like JavaScript — the highest-value scan + seeds, since bundled JS is where secrets most often leak.""" + return [u for u in self.urls if urlparse(u).path.lower().endswith(".js")] + + def to_dict(self) -> dict: + return { + "domain": self.domain, + "urls": self.urls, + "count": self.count, + "paths": self.paths, + "js_urls": self.js_urls(), + "sources": self.sources, + "error": self.error, + } + + +def _in_scope(url: str, domain: str) -> bool: + """True if `url`'s host is `domain` or a subdomain of it.""" + try: + host = (urlparse(url).hostname or "").lower() + except ValueError: + return False + return bool(host) and (host == domain or host.endswith("." + domain)) + + +def parse_wayback_cdx(payload: object, domain: str) -> list[str]: + """Parse a Wayback CDX `output=json&fl=original` response into sorted, in-scope + URLs. The response is a JSON array whose first row is the header ['original']. + Pure function.""" + domain = (domain or "").strip().lower().rstrip(".") + if isinstance(payload, (str, bytes)): + try: + payload = json.loads(payload) + except (ValueError, TypeError): + return [] + if not isinstance(payload, list): + return [] + + found: set[str] = set() + for row in payload: + if not isinstance(row, list) or not row: + continue + url = str(row[0]).strip() + if not url or url == "original": # skip the CDX header row + continue + if _in_scope(url, domain): + found.add(url) + return sorted(found) + + +def parse_commoncrawl_jsonl(text: str, domain: str) -> list[str]: + """Parse a CommonCrawl CDX JSONL response (one JSON object per line, each with + a `url` field) into sorted, in-scope URLs. Pure function.""" + domain = (domain or "").strip().lower().rstrip(".") + found: set[str] = set() + for line in (text or "").splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except (ValueError, TypeError): + continue + url = str(record.get("url", "")).strip() if isinstance(record, dict) else "" + if url and _in_scope(url, domain): + found.add(url) + return sorted(found) + + +async def _get_with_retries( + client: httpx.AsyncClient, url: str, +) -> tuple[httpx.Response | None, str | None]: + """GET with backoff retries on transient statuses/timeouts. Returns + (response, None) or (None, error-with-type-name).""" + last_err = "no attempt made" + for attempt in range(HISTORICAL_RETRIES + 1): + try: + resp = await client.get( + url, timeout=httpx.Timeout(HISTORICAL_TIMEOUT, connect=10.0), + ) + if resp.status_code in _TRANSIENT_STATUS and attempt < HISTORICAL_RETRIES: + last_err = f"HTTP {resp.status_code}" + await asyncio.sleep(2 ** attempt) + continue + return resp, None + except (httpx.HTTPError, httpx.InvalidURL) as exc: + last_err = f"{type(exc).__name__}: {exc}".rstrip(": ").strip() + if attempt < HISTORICAL_RETRIES: + await asyncio.sleep(2 ** attempt) + continue + return None, last_err + + +async def fetch_wayback( + client: httpx.AsyncClient, domain: str, *, limit: int, +) -> tuple[set[str], str | None]: + # matchType=domain covers the apex AND every subdomain in the archive. + url = (f"{WAYBACK_CDX_URL}?url={domain}&matchType=domain&output=json" + f"&fl=original&collapse=urlkey&limit={limit}") + resp, err = await _get_with_retries(client, url) + if resp is None: + return set(), err + if resp.status_code != 200: + return set(), f"HTTP {resp.status_code}" + return set(parse_wayback_cdx(resp.text, domain)), None + + +async def fetch_commoncrawl( + client: httpx.AsyncClient, domain: str, *, limit: int, +) -> tuple[set[str], str | None]: + """CommonCrawl needs two hops: discover the newest index's CDX API, then query + it. Both fail closed.""" + resp, err = await _get_with_retries(client, COMMONCRAWL_COLLINFO) + if resp is None: + return set(), err + if resp.status_code != 200: + return set(), f"collinfo HTTP {resp.status_code}" + try: + indexes = json.loads(resp.text) + cdx_api = indexes[0]["cdx-api"] # newest index is first + except (ValueError, TypeError, KeyError, IndexError) as exc: + return set(), f"collinfo parse: {type(exc).__name__}" + + resp2, err2 = await _get_with_retries( + client, f"{cdx_api}?url={domain}&matchType=domain&output=json&limit={limit}") + if resp2 is None: + return set(), err2 + if resp2.status_code != 200: + return set(), f"HTTP {resp2.status_code}" + return set(parse_commoncrawl_jsonl(resp2.text, domain)), None + + +async def discover_historical_urls( + client: httpx.AsyncClient, + domain: str, + *, + limit: int = MAX_HISTORICAL_URLS, + enable_commoncrawl: bool = ENABLE_COMMONCRAWL, +) -> HistoricalResult: + """Recover a domain's historically-exposed URLs from public archives + (Wayback + optionally CommonCrawl), merged and deduplicated. + + Never contacts the target. Fails closed: `error` is set only when every + enabled source fails; if any source returns data the result is usable.""" + domain = (domain or "").strip().lower().rstrip(".") + if not domain: + return HistoricalResult(domain=domain, error="empty domain") + + sources: list[tuple[str, object]] = [("wayback", fetch_wayback)] + if enable_commoncrawl: + sources.append(("commoncrawl", fetch_commoncrawl)) + + all_urls: set[str] = set() + ok_sources: list[str] = [] + errors: list[str] = [] + for name, fetch in sources: + try: + urls, err = await fetch(client, domain, limit=limit) + except Exception as exc: # a source must never crash the run + urls, err = set(), f"{type(exc).__name__}: {exc}".rstrip(": ").strip() + if urls: + all_urls |= urls + ok_sources.append(name) + if err: + errors.append(f"{name}: {err}") + + urls = sorted(all_urls) + if len(urls) > limit: + urls = urls[:limit] + + error = None if ok_sources else ("; ".join(errors) or "no sources returned data") + return HistoricalResult(domain=domain, urls=urls, sources=ok_sources, error=error) diff --git a/backend/tests/test_historical.py b/backend/tests/test_historical.py new file mode 100644 index 0000000..759df18 --- /dev/null +++ b/backend/tests/test_historical.py @@ -0,0 +1,153 @@ +"""Tests for historical path discovery (deep-ASM slice 3): Wayback + CommonCrawl. +Network is mocked; the pure parsers are exercised directly.""" + +from __future__ import annotations + +import json + +import httpx +import pytest + +import historical + + +class TestParseWaybackCdx: + def test_skips_header_and_keeps_in_scope(self): + payload = [ + ["original"], # CDX header row + ["https://example.com/admin"], + ["https://api.example.com/v1/keys"], + ["https://evil.com/example.com"], # out of scope + ] + out = historical.parse_wayback_cdx(payload, "example.com") + assert out == ["https://api.example.com/v1/keys", "https://example.com/admin"] + + def test_accepts_raw_json_and_bad_input(self): + raw = json.dumps([["original"], ["https://example.com/x"]]) + assert historical.parse_wayback_cdx(raw, "example.com") == ["https://example.com/x"] + assert historical.parse_wayback_cdx("nope", "example.com") == [] + assert historical.parse_wayback_cdx({"bad": 1}, "example.com") == [] + + +class TestParseCommoncrawlJsonl: + def test_parses_jsonl_and_filters_scope(self): + text = "\n".join([ + json.dumps({"url": "https://example.com/old.js"}), + json.dumps({"url": "https://shop.example.com/checkout"}), + json.dumps({"url": "https://other.net/x"}), # out of scope + "not-json-line", # tolerated + ]) + out = historical.parse_commoncrawl_jsonl(text, "example.com") + assert out == ["https://example.com/old.js", "https://shop.example.com/checkout"] + + +class TestHistoricalResultViews: + def test_paths_and_js_urls(self): + r = historical.HistoricalResult(domain="example.com", urls=[ + "https://example.com/a", "https://example.com/a?x=1", + "https://example.com/static/app.js", "https://example.com/b", + ]) + # /a and /a?x=1 collapse to one path. + assert r.paths == ["/a", "/b", "/static/app.js"] + assert r.js_urls() == ["https://example.com/static/app.js"] + + +class _Resp: + def __init__(self, status_code: int, text: str = ""): + self.status_code = status_code + self.text = text + + +class _FakeClient: + """Routes GETs by URL substring; a route value may be a response, an + Exception, or a list consumed in order (for retry tests).""" + def __init__(self, routes: dict): + self._routes = {k: (list(v) if isinstance(v, list) else v) for k, v in routes.items()} + self.requested_urls: list[str] = [] + + async def get(self, url, **_kw): + self.requested_urls.append(url) + val = next((v for k, v in self._routes.items() if k in url), _Resp(404)) + item = (val.pop(0) if len(val) > 1 else val[0]) if isinstance(val, list) else val + if isinstance(item, Exception): + raise item + return item + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + async def _fast(*_a, **_k): + return None + monkeypatch.setattr(historical.asyncio, "sleep", _fast) + + +def _wayback_body(*urls: str) -> str: + return json.dumps([["original"], *[[u] for u in urls]]) + + +@pytest.mark.asyncio +async def test_discover_wayback_only(): + client = _FakeClient({"web.archive.org": _Resp(200, _wayback_body( + "https://example.com/admin", "https://example.com/app.js"))}) + result = await historical.discover_historical_urls( + client, "example.com", enable_commoncrawl=False) + assert result.error is None + assert result.urls == ["https://example.com/admin", "https://example.com/app.js"] + assert result.sources == ["wayback"] + assert result.js_urls() == ["https://example.com/app.js"] + + +@pytest.mark.asyncio +async def test_discover_merges_wayback_and_commoncrawl(): + collinfo = json.dumps([{"id": "CC-MAIN-latest", "cdx-api": "https://index.commoncrawl.org/CC-MAIN-latest-index"}]) + client = _FakeClient({ + "web.archive.org": _Resp(200, _wayback_body("https://example.com/a")), + "collinfo.json": _Resp(200, collinfo), + "CC-MAIN-latest-index": _Resp(200, json.dumps({"url": "https://example.com/b"})), + }) + result = await historical.discover_historical_urls(client, "example.com") + assert result.urls == ["https://example.com/a", "https://example.com/b"] + assert set(result.sources) == {"wayback", "commoncrawl"} + + +@pytest.mark.asyncio +async def test_discover_survives_one_source_down(): + client = _FakeClient({ + "web.archive.org": _Resp(200, _wayback_body("https://example.com/a")), + "collinfo.json": _Resp(503), # CommonCrawl down + }) + result = await historical.discover_historical_urls(client, "example.com") + assert result.urls == ["https://example.com/a"] + assert result.sources == ["wayback"] + assert result.error is None # one good source → not an error + + +@pytest.mark.asyncio +async def test_discover_retries_transient_then_succeeds(): + client = _FakeClient({ + "web.archive.org": [_Resp(502), _Resp(200, _wayback_body("https://example.com/a"))], + }) + result = await historical.discover_historical_urls( + client, "example.com", enable_commoncrawl=False) + assert result.urls == ["https://example.com/a"] + + +@pytest.mark.asyncio +async def test_discover_all_fail_sets_error(): + client = _FakeClient({ + "web.archive.org": httpx.ConnectError("boom"), + "collinfo.json": _Resp(503), + }) + result = await historical.discover_historical_urls(client, "example.com") + assert result.urls == [] + assert result.error is not None + assert "wayback" in result.error and "commoncrawl" in result.error + + +@pytest.mark.asyncio +async def test_discover_respects_limit(): + urls = [f"https://example.com/p{i}" for i in range(10)] + client = _FakeClient({"web.archive.org": _Resp(200, _wayback_body(*urls))}) + result = await historical.discover_historical_urls( + client, "example.com", limit=3, enable_commoncrawl=False) + assert result.count == 3 From b24d68ba5e20c6059e3197e784e2f8134a55706b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 19:28:09 +0000 Subject: [PATCH 09/19] Feed historical JS bundles into the scan (deep-ASM slice 3.5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns discovery into findings — the payoff of the passive discovery chain. A secret sitting in a forgotten/archived JS bundle that no live page links to now gets fetched and confirmed. • scanner.run_scan() gains seed_urls: externally-supplied asset URLs are fetched (reusing fetch_url) and added to the scan set after the live crawl, deduped against it and capped by MAX_SEED_URLS. Backward- compatible (defaults to none). • orchestrator.run_deep_scan(include_historical=True) recovers the domain's historical URLs once (Wayback/CommonCrawl) and routes each host its own archived JS bundles as seeds. discover_historical_fn is injected for testing; DeepScanResult carries a historical_urls count. • CLI: `--deep-scan --with-historical`. Combined report gains a "Historical URLs" stat; deep-scan summary line reports the count. Tests: run_scan seed fetch + dedup + no-seed no-op (collaborators mocked); orchestrator routes per-host seeds and reports the count, and passes none when not requested. Suite 239 -> 244, ruff clean. Passive posture intact. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- .env.example | 3 ++ CHANGELOG.md | 8 +++++ backend/cli.py | 8 ++++- backend/orchestrator.py | 20 +++++++++++- backend/report.py | 1 + backend/scanner.py | 30 ++++++++++++++++++ backend/tests/test_orchestrator.py | 51 ++++++++++++++++++++++++++++++ backend/tests/test_scanner.py | 46 +++++++++++++++++++++++++++ 8 files changed, 165 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 7f7bf26..13e9f32 100644 --- a/.env.example +++ b/.env.example @@ -90,6 +90,9 @@ HISTORICAL_TIMEOUT=30 HISTORICAL_RETRIES=2 # Safety cap on how many historical URLs to return. MAX_HISTORICAL_URLS=2000 +# Cap on externally-supplied seed assets fetched per scan (e.g. historical JS +# bundles fed into a --deep-scan --with-historical run). +MAX_SEED_URLS=200 # Skip JS assets larger than this many bytes (default 5 MB). MAX_ASSET_BYTES=5242880 # Only surface AI-confirmed findings at or above this confidence (0-100). diff --git a/CHANGELOG.md b/CHANGELOG.md index 3008b4e..9bcaff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ All notable changes to SecretNode are documented here. This project adheres to ## [Unreleased] ### Added +- **Historical bundles fed into the scan (deep-ASM slice 3.5).** `run_scan()` gains a `seed_urls` + parameter — externally-supplied asset URLs are fetched and scanned alongside the live crawl, + deduped against it (capped by `MAX_SEED_URLS`). `run_deep_scan(include_historical=True)` now + recovers the domain's historical JS bundles (Wayback/CommonCrawl) and routes each host its own + archived bundles as seeds, so a secret in a forgotten bundle **no live page links to** still gets + fetched and confirmed. CLI: `python cli.py --deep-scan --with-historical`; the combined + report gains a "Historical URLs" metric. This turns discovery into findings — the payoff of the + whole passive discovery chain. - **Historical path discovery (deep-ASM slice 3).** New `backend/historical.py` recovers a domain's historically-exposed URLs from **public web archives (Wayback Machine + CommonCrawl)** — the passive alternative to directory/content brute-forcing, so no request ever touches the target. Two diff --git a/backend/cli.py b/backend/cli.py index 5ad1f09..fcca0ae 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -94,6 +94,10 @@ def build_parser() -> argparse.ArgumentParser: "scan each, and write a combined report. Passive; authorized use only.") ap.add_argument("--max-targets", dest="max_targets", type=int, default=orchestrator.MAX_TARGETS, help=f"Max live hosts to scan in a --deep-scan run (default: {orchestrator.MAX_TARGETS})") + ap.add_argument("--with-historical", dest="with_historical", action="store_true", + help="In a --deep-scan, also recover historical JS bundles from public archives " + "(Wayback/CommonCrawl) and scan them as seeds — catches secrets in forgotten " + "bundles no live page links to. Slower; passive.") return ap @@ -105,6 +109,7 @@ async def _run_deep_scan(args) -> int: verify=args.verify, only_verified=args.only_verified, max_targets=max(1, args.max_targets), + include_historical=args.with_historical, ) if args.output: report_fmt = (args.format if args.format in ("json",) else "html") @@ -117,9 +122,10 @@ async def _run_deep_scan(args) -> int: sys.stdout.write(json.dumps(result.to_dict(), indent=2) + "\n") t = result.to_dict()["totals"] + hist = f", {t['historical_urls']} historical URL(s)" if t.get("historical_urls") else "" print( f"SecretNode deep scan of {result.domain}: {t['subdomains']} subdomain(s), " - f"{t['live_hosts']} live, {t['hosts_scanned']} scanned — " + f"{t['live_hosts']} live, {t['hosts_scanned']} scanned{hist} — " f"{t['confirmed']} confirmed, {t['needs_review']} needs-review, " f"{t['posture_issues']} posture issue(s).", file=sys.stderr, diff --git a/backend/orchestrator.py b/backend/orchestrator.py index f6648be..a17b7e1 100644 --- a/backend/orchestrator.py +++ b/backend/orchestrator.py @@ -31,6 +31,7 @@ import httpx +import historical import recon import scanner @@ -82,6 +83,7 @@ class DeepScanResult: live_hosts: list[str] = field(default_factory=list) hosts: list[HostScan] = field(default_factory=list) scans: list[dict] = field(default_factory=list) # raw per-host scan dicts + historical_urls: int = 0 # historical URLs discovered (0 if not requested) error: str | None = None @property @@ -103,10 +105,12 @@ def to_dict(self) -> dict: "enum_sources": self.enum_sources, "live_hosts": self.live_hosts, "hosts": [h.to_dict() for h in self.hosts], + "historical_urls": self.historical_urls, "totals": { "subdomains": len(self.subdomains), "live_hosts": len(self.live_hosts), "hosts_scanned": len(self.hosts), + "historical_urls": self.historical_urls, "confirmed": self.total_confirmed, "needs_review": self.total_needs_review, "posture_issues": self.total_posture, @@ -179,9 +183,12 @@ async def run_deep_scan( verify: bool = False, only_verified: bool = False, max_targets: int = MAX_TARGETS, + include_historical: bool = False, enumerate_fn: EnumerateFn = recon.enumerate_subdomains, scan_fn: ScanFn = scanner.run_scan, client_factory: ClientFactory = scanner.build_client, + discover_historical_fn: Callable[..., Awaitable["historical.HistoricalResult"]] + = historical.discover_historical_urls, ) -> DeepScanResult: """Domain → enumerate → probe → scan each live host → aggregate. @@ -230,12 +237,23 @@ async def run_deep_scan( result.error = enum.error or "no live hosts found" return result + # Optional: recover historical URLs (Wayback/CommonCrawl) once for the + # domain and hand each host its archived JS bundles as scan seeds — so a + # forgotten bundle no page links to still gets fetched and scanned. + js_by_host: dict[str, list[str]] = {} + if include_historical: + hist = await discover_historical_fn(client, domain) + result.historical_urls = hist.count + for u in hist.js_urls(): + js_by_host.setdefault(recon._host_of(u), []).append(u) + targets = result.live_hosts[:max(1, max_targets)] for url in targets: host = recon._host_of(url) try: scan = await scan_fn(target_url=url, max_crawl_pages=max_crawl_pages, - verify=verify, only_verified=only_verified) + verify=verify, only_verified=only_verified, + seed_urls=js_by_host.get(host, [])) except Exception as exc: # a single host must not sink the whole run logger.debug("deep scan: host %s failed: %s", url, exc) result.hosts.append(HostScan(host=host, url=url, diff --git a/backend/report.py b/backend/report.py index 966c7a7..6a77f05 100644 --- a/backend/report.py +++ b/backend/report.py @@ -619,6 +619,7 @@ def host_row(h: dict[str, Any]) -> str:
{int(totals.get('subdomains', 0))}
Subdomains found
{int(totals.get('live_hosts', 0))}
Live hosts
{int(totals.get('hosts_scanned', 0))}
Hosts scanned
+
{int(totals.get('historical_urls', 0))}
Historical URLs
{confirmed_total}
Confirmed exposures
{int(totals.get('needs_review', 0))}
Needs review
{int(totals.get('posture_issues', 0))}
Posture issues
diff --git a/backend/scanner.py b/backend/scanner.py index d5dcc79..60c3203 100644 --- a/backend/scanner.py +++ b/backend/scanner.py @@ -104,6 +104,9 @@ def _env_float(name: str, default: float) -> float: MAX_MATCHES_PER_PATTERN = _env_int("MAX_MATCHES_PER_PATTERN", 100) # R3 defence-in-depth: bound the # matches examined for ANY single pattern on ANY single text, so a # crafted blob cannot spawn millions of matches for one detector. +MAX_SEED_URLS = _env_int("MAX_SEED_URLS", 200) # cap externally-supplied seed assets fetched per scan + # (e.g. historical JS bundles from public archives) — bounds the + # extra fetches a deep scan does beyond the live crawl. # ── Type alias for the broadcaster callback ──────────────────────────────────── Broadcaster = Callable[[dict[str, Any]], Coroutine[Any, Any, None]] @@ -1770,6 +1773,7 @@ async def run_scan( max_crawl_pages: int = 1, verify: bool | None = None, only_verified: bool = False, + seed_urls: list[str] | None = None, ) -> dict[str, Any]: """ Full pipeline: @@ -1840,6 +1844,32 @@ async def emit(event: dict[str, Any]) -> None: await emit({"type": "scan_error", "error": str(exc)}) return result + # ── 1a. Inject seed assets (deep-ASM slice 3.5) ──────────────────── + # Externally-supplied URLs — e.g. historical JS bundles recovered from + # public archives (Wayback/CommonCrawl) — that the live crawl would never + # link to. Fetch any not already collected and add them to the scan set. + if seed_urls: + state.check() + have = {u for u, _ in assets} + to_fetch = [u for u in dict.fromkeys(seed_urls) if u not in have][:MAX_SEED_URLS] + if to_fetch: + await emit({ + "type": "log", "level": "INFO", + "message": f"Fetching {len(to_fetch)} seed asset(s) from archives", + }) + fetched = await asyncio.gather( + *(fetch_url(client, u, semaphore, broadcast) for u in to_fetch) + ) + added = 0 + for u, body in fetched: + if body: + assets.append((u, body)) + added += 1 + await emit({ + "type": "log", "level": "INFO", + "message": f"Added {added} seed asset(s) from archives ({len(to_fetch) - added} unreachable)", + }) + result["assets_fetched"] = len(assets) await emit({ "type": "log", "level": "INFO", diff --git a/backend/tests/test_orchestrator.py b/backend/tests/test_orchestrator.py index 4335358..bd206af 100644 --- a/backend/tests/test_orchestrator.py +++ b/backend/tests/test_orchestrator.py @@ -7,6 +7,7 @@ import httpx import pytest +import historical import orchestrator import recon @@ -165,6 +166,56 @@ async def test_deep_scan_ip_target_falls_back_to_single_scan(): assert result.hosts[0].url == "http://93.184.216.34" +@pytest.mark.asyncio +async def test_deep_scan_historical_seeds_routed_per_host(): + seen: dict[str, list[str]] = {} + + async def _scan(*, target_url, seed_urls=None, **_kw): + seen[target_url] = list(seed_urls or []) + return {"target_url": target_url, "confirmed_findings": [], + "needs_review_findings": [], "posture_findings": [], "assets_fetched": 1} + + async def _hist(_client, domain): + return historical.HistoricalResult( + domain=domain, + urls=["https://a.example.com/old.js", "https://example.com/x.js", + "https://example.com/page.html"], # non-JS ignored as a seed + sources=["wayback"], + ) + + result = await orchestrator.run_deep_scan( + "example.com", + include_historical=True, + enumerate_fn=_enum_returning(["a.example.com"]), + scan_fn=_scan, + client_factory=_client_factory(), + discover_historical_fn=_hist, + ) + # Each host receives only its own archived JS bundles as seeds. + assert seen["https://example.com"] == ["https://example.com/x.js"] + assert seen["https://a.example.com"] == ["https://a.example.com/old.js"] + assert result.historical_urls == 3 + assert result.to_dict()["totals"]["historical_urls"] == 3 + + +@pytest.mark.asyncio +async def test_deep_scan_without_historical_passes_no_seeds(): + seen: dict[str, list[str]] = {} + + async def _scan(*, target_url, seed_urls=None, **_kw): + seen[target_url] = list(seed_urls or []) + return {"target_url": target_url, "confirmed_findings": [], + "needs_review_findings": [], "posture_findings": [], "assets_fetched": 1} + + await orchestrator.run_deep_scan( + "example.com", + enumerate_fn=_enum_returning([]), + scan_fn=_scan, + client_factory=_client_factory(), + ) + assert seen["https://example.com"] == [] + + @pytest.mark.asyncio async def test_deep_scan_ssrf_guard_skips_private_host(monkeypatch): # Turn the guard back on and force one host to look internal. diff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.py index 41db67d..eba8153 100644 --- a/backend/tests/test_scanner.py +++ b/backend/tests/test_scanner.py @@ -261,6 +261,52 @@ def test_generic_ai_rejection_dropped_even_if_uncertain(self): assert scanner.classify_validated(self._vf("Generic High-Entropy Secret", False, 55)) == "drop" +class TestSeedUrlInjection: + """run_scan(seed_urls=...) must fetch externally-supplied assets (e.g. archived + JS bundles) and add them to the scan set, deduped against the live crawl.""" + + class _FakeClient: + async def __aenter__(self): + return self + + async def __aexit__(self, *_a): + return False + + def _patch(self, monkeypatch): + async def fake_spider(client, url, sem, broadcast, max_pages=1): + return [("https://ex.com/base.js", "const base = 1;")] + + async def fake_fetch(client, url, sem, broadcast=None): + return (url, "// clean seed body, no secrets") + + async def fake_posture(client, url): + return [] + + monkeypatch.setattr(scanner, "spider_target", fake_spider) + monkeypatch.setattr(scanner, "fetch_url", fake_fetch) + monkeypatch.setattr(scanner, "build_client", lambda *a, **k: self._FakeClient()) + monkeypatch.setattr(scanner.posture, "fetch_posture", fake_posture) + + @pytest.mark.asyncio + async def test_seed_url_is_fetched_and_added(self, monkeypatch): + self._patch(monkeypatch) + res = await scanner.run_scan("https://ex.com", seed_urls=["https://ex.com/old.js"]) + assert res["assets_fetched"] == 2 # base crawl asset + 1 seed + + @pytest.mark.asyncio + async def test_seed_url_deduped_against_crawl(self, monkeypatch): + self._patch(monkeypatch) + # Seed already collected by the crawl → not fetched twice. + res = await scanner.run_scan("https://ex.com", seed_urls=["https://ex.com/base.js"]) + assert res["assets_fetched"] == 1 + + @pytest.mark.asyncio + async def test_no_seeds_is_unchanged(self, monkeypatch): + self._patch(monkeypatch) + res = await scanner.run_scan("https://ex.com") + assert res["assets_fetched"] == 1 + + class TestNeedsReviewSentinel: def test_sentinel_is_negative(self): # Must never collide with a real 0-100 confidence value From b4c017c79d685ac3a88a0a74a44678b70cd37b26 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 19:34:08 +0000 Subject: [PATCH 10/19] Fix deep-scan dropping the specified host; let historical reveal hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live testphp.vulnweb.com --deep-scan returned 0 hosts: the target was reduced to the registrable domain vulnweb.com, CT enumeration returned nothing that run, the bare apex was not live, and the host the user actually typed (testphp.vulnweb.com) was never scanned. • Always include the originally-specified host as the first scan candidate — enumeration can never drop the target the caller named. • When include_historical, harvest hostnames from the recovered archive URLs and add them as candidates too (not just JS seeds). Historical now enriches HOST discovery as well as paths, so a flaky/empty CT source no longer zeroes out a run — the archive's testphp/testasp/ rest/www hosts get scanned. Candidates are filtered in-scope and deduped; historical discovery moved ahead of host selection. Tests: specified host scanned despite empty CT enumeration; archive-only hostnames become scanned candidates. Suite 244 -> 246, ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- backend/orchestrator.py | 36 ++++++++++++++------- backend/tests/test_orchestrator.py | 50 ++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/backend/orchestrator.py b/backend/orchestrator.py index a17b7e1..f061877 100644 --- a/backend/orchestrator.py +++ b/backend/orchestrator.py @@ -208,13 +208,35 @@ async def run_deep_scan( result.scans = [scan] return result + original_host = recon._host_of(target) result = DeepScanResult(domain=domain) async with client_factory() as client: enum = await enumerate_fn(client, domain) result.subdomains = enum.subdomains result.enum_sources = enum.sources - # Always include the apex domain as a candidate even if CT omitted it. - candidates = list(dict.fromkeys([domain, *enum.subdomains])) + + # Optional: recover historical URLs (Wayback/CommonCrawl) once for the + # domain. They enrich BOTH host discovery (hostnames seen in the archive) + # and per-host scan seeds (archived JS bundles) — so a flaky CT source no + # longer zeroes out the run, and forgotten bundles still get scanned. + js_by_host: dict[str, list[str]] = {} + hist_hosts: list[str] = [] + if include_historical: + hist = await discover_historical_fn(client, domain) + result.historical_urls = hist.count + hist_hosts = [recon._host_of(u) for u in hist.urls] + for u in hist.js_urls(): + js_by_host.setdefault(recon._host_of(u), []).append(u) + + # Candidate hosts, in-scope and deduped. The host the caller actually + # typed is ALWAYS included first — enumeration must never be able to drop + # the specified target — followed by the apex, CT subdomains, and any + # hosts seen in the archive. + candidates = [ + h for h in dict.fromkeys( + [original_host, domain, *enum.subdomains, *hist_hosts]) + if h and (h == domain or h.endswith("." + domain)) + ] # SSRF guard: never probe/scan a discovered host that resolves to an # internal address (a wildcard/misissued cert can name one). Bypassed @@ -237,16 +259,6 @@ async def run_deep_scan( result.error = enum.error or "no live hosts found" return result - # Optional: recover historical URLs (Wayback/CommonCrawl) once for the - # domain and hand each host its archived JS bundles as scan seeds — so a - # forgotten bundle no page links to still gets fetched and scanned. - js_by_host: dict[str, list[str]] = {} - if include_historical: - hist = await discover_historical_fn(client, domain) - result.historical_urls = hist.count - for u in hist.js_urls(): - js_by_host.setdefault(recon._host_of(u), []).append(u) - targets = result.live_hosts[:max(1, max_targets)] for url in targets: host = recon._host_of(url) diff --git a/backend/tests/test_orchestrator.py b/backend/tests/test_orchestrator.py index bd206af..43e099c 100644 --- a/backend/tests/test_orchestrator.py +++ b/backend/tests/test_orchestrator.py @@ -216,6 +216,56 @@ async def _scan(*, target_url, seed_urls=None, **_kw): assert seen["https://example.com"] == [] +@pytest.mark.asyncio +async def test_deep_scan_always_scans_specified_host(): + # Regression: even if CT enumeration returns nothing and the typed host is a + # subdomain (not the apex), the host the caller specified must still be scanned. + seen: list[str] = [] + + async def _scan(*, target_url, seed_urls=None, **_kw): + seen.append(target_url) + return {"target_url": target_url, "confirmed_findings": [], + "needs_review_findings": [], "posture_findings": [], "assets_fetched": 1} + + await orchestrator.run_deep_scan( + "testphp.example.com", + enumerate_fn=_enum_returning([]), # CT found nothing + scan_fn=_scan, + client_factory=_client_factory(), + ) + assert any("testphp.example.com" in u for u in seen) + + +@pytest.mark.asyncio +async def test_deep_scan_historical_reveals_hosts(): + # Hostnames seen only in the archive become scan candidates, so a flaky CT + # source no longer zeroes out the run. + seen: list[str] = [] + + async def _scan(*, target_url, seed_urls=None, **_kw): + seen.append(recon._host_of(target_url)) + return {"target_url": target_url, "confirmed_findings": [], + "needs_review_findings": [], "posture_findings": [], "assets_fetched": 1} + + async def _hist(_client, domain): + return historical.HistoricalResult( + domain=domain, + urls=["https://testphp.example.com/x", "https://rest.example.com/b.js"], + sources=["wayback"], + ) + + await orchestrator.run_deep_scan( + "example.com", + include_historical=True, + enumerate_fn=_enum_returning([]), # CT empty + scan_fn=_scan, + client_factory=_client_factory(), + discover_historical_fn=_hist, + ) + assert "testphp.example.com" in seen + assert "rest.example.com" in seen + + @pytest.mark.asyncio async def test_deep_scan_ssrf_guard_skips_private_host(monkeypatch): # Turn the guard back on and force one host to look internal. From ede764c87fcfe834e23b91f58670881b1320f482 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 19:40:32 +0000 Subject: [PATCH 11/19] Deep-scan report: show the actual findings, not just per-host counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The combined report summarised each host's confirmed/needs-review as numbers but never listed the findings, so a run reporting "22 needs- review" gave the operator no way to actually review them. • DeepScanResult.to_dict() now aggregates confirmed_findings and needs_review_findings across all hosts, tagging each with its host of origin (_host) for provenance. • generate_deep_scan_html() renders two new detail tables — "Confirmed Findings (all hosts)" and "Flagged for Manual Review (all hosts)" — with host, severity, type, location and reasoning, capped at 250 rows each. Adds severity-badge styling. Now the deliverable is complete: the needs-review candidates a deep scan surfaces are inspectable, not just counted. Suite 246 -> 247, ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- backend/orchestrator.py | 12 +++++++ backend/report.py | 55 ++++++++++++++++++++++++++++++ backend/tests/test_orchestrator.py | 31 +++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/backend/orchestrator.py b/backend/orchestrator.py index f061877..42da1fc 100644 --- a/backend/orchestrator.py +++ b/backend/orchestrator.py @@ -98,6 +98,16 @@ def total_needs_review(self) -> int: def total_posture(self) -> int: return sum(h.posture_issues for h in self.hosts) + def _aggregate(self, key: str) -> list[dict]: + """Flatten a per-host finding list across all scans, tagging each finding + with the host it came from so the combined report can show provenance.""" + out: list[dict] = [] + for scan in self.scans: + host = recon._host_of(scan.get("target_url", "")) or scan.get("target_url", "") + for f in scan.get(key, []): + out.append({**f, "_host": host}) + return out + def to_dict(self) -> dict: return { "domain": self.domain, @@ -106,6 +116,8 @@ def to_dict(self) -> dict: "live_hosts": self.live_hosts, "hosts": [h.to_dict() for h in self.hosts], "historical_urls": self.historical_urls, + "confirmed_findings": self._aggregate("confirmed_findings"), + "needs_review_findings": self._aggregate("needs_review_findings"), "totals": { "subdomains": len(self.subdomains), "live_hosts": len(self.live_hosts), diff --git a/backend/report.py b/backend/report.py index 6a77f05..f1aadb1 100644 --- a/backend/report.py +++ b/backend/report.py @@ -580,6 +580,42 @@ def host_row(h: dict[str, Any]) -> str: 'No live hosts were scanned.' subs = ", ".join(html.escape(s) for s in deep.get("subdomains", [])) or "—" + # Findings aggregated across every host, each tagged with its host of origin. + _MAX_ROWS = 250 + confirmed_findings = deep.get("confirmed_findings", [])[:_MAX_ROWS] + review_findings = deep.get("needs_review_findings", [])[:_MAX_ROWS] + + def _sev(f: dict[str, Any]) -> str: + s = str(f.get("severity", "MEDIUM")).upper() + return f'{html.escape(s)}' + + def _loc(f: dict[str, Any]) -> str: + return html.escape(str(f.get("source_url", f.get("target_url", "")))) + + def conf_row(f: dict[str, Any]) -> str: + return ("" + f'{html.escape(str(f.get("_host", "")))}' + f"{_sev(f)}" + f'{html.escape(str(f.get("secret_type", "")))}' + f'{_loc(f)}' + f'{int(f.get("confidence", 0) or 0)}%' + f'{html.escape(str(f.get("reason", "")))}' + "") + + def review_row(f: dict[str, Any]) -> str: + return ("" + f'{html.escape(str(f.get("_host", "")))}' + f"{_sev(f)}" + f'{html.escape(str(f.get("secret_type", "")))}' + f'{_loc(f)}' + f'{html.escape(str(f.get("reason", "")))}' + "") + + conf_rows = "\n".join(conf_row(f) for f in confirmed_findings) or \ + 'No confirmed credential exposures.' + review_rows = "\n".join(review_row(f) for f in review_findings) or \ + 'None.' + return f""" Domain Attack-Surface Report — {domain} @@ -596,6 +632,11 @@ def host_row(h: dict[str, Any]) -> str: .empty {{ text-align: center; color: #718096; font-style: italic; padding: 20px; }} .hit {{ color: #c53030; font-weight: bold; }} .err {{ color: #dd6b20; font-weight: bold; }} + .sev {{ padding: 2px 8px; border-radius: 4px; font-size: 10px; font-weight: bold; color: #fff; }} + .sev-critical {{ background: #c53030; }} + .sev-high {{ background: #dd6b20; }} + .sev-medium {{ background: #d69e2e; }} + .sev-low {{ background: #3182ce; }} .verdict {{ background: #f7fafc; border: 1px solid #e2e8f0; border-left: 6px solid {pill_color}; border-radius: 6px; padding: 16px 18px; margin: 18px 0; }} .risk-pill {{ color: #fff; font-weight: bold; font-size: 12px; letter-spacing: 0.06em; @@ -631,6 +672,20 @@ def host_row(h: dict[str, Any]) -> str: {rows} +

Confirmed Findings (all hosts)

+ + + {conf_rows} +
HostSeverityTypeLocationConfidenceAI Reasoning
+ +

Flagged for Manual Review (all hosts)

+
Candidates a human should confirm — a structural match the AI could not + confidently clear, or a scan where AI validation was unavailable. Not confirmed exposures.
+ + + {review_rows} +
HostSeverityTypeLocationNote
+

Discovered subdomain surface

{subs}
diff --git a/backend/tests/test_orchestrator.py b/backend/tests/test_orchestrator.py index 43e099c..a741b3a 100644 --- a/backend/tests/test_orchestrator.py +++ b/backend/tests/test_orchestrator.py @@ -216,6 +216,37 @@ async def _scan(*, target_url, seed_urls=None, **_kw): assert seen["https://example.com"] == [] +@pytest.mark.asyncio +async def test_deep_scan_aggregates_findings_with_host_and_renders(): + import report + + async def _scan(*, target_url, seed_urls=None, **_kw): + host = recon._host_of(target_url) + conf = ([{"secret_type": "AWS Access Key", "severity": "CRITICAL"}] + if host.startswith("a.") else []) + rev = [{"secret_type": "Generic", "severity": "MEDIUM", "reason": "unclear"}] + return {"target_url": target_url, "confirmed_findings": conf, + "needs_review_findings": rev, "posture_findings": [], "assets_fetched": 1} + + result = await orchestrator.run_deep_scan( + "example.com", + enumerate_fn=_enum_returning(["a.example.com"]), + scan_fn=_scan, + client_factory=_client_factory(), + ) + d = result.to_dict() + # Confirmed finding is tagged with the host it came from. + assert any(f["_host"] == "a.example.com" and f["secret_type"] == "AWS Access Key" + for f in d["confirmed_findings"]) + # Needs-review aggregated across both hosts (apex + a), each host-tagged. + assert len(d["needs_review_findings"]) == 2 + assert all("_host" in f for f in d["needs_review_findings"]) + # The combined report actually renders the detail, not just counts. + htmlrep = report.generate_deep_scan_html(d) + assert "Flagged for Manual Review (all hosts)" in htmlrep + assert "AWS Access Key" in htmlrep and "a.example.com" in htmlrep + + @pytest.mark.asyncio async def test_deep_scan_always_scans_specified_host(): # Regression: even if CT enumeration returns nothing and the typed host is a From 7d0bb2777651f8266ec82e8140357a83a3d73baf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 20:00:42 +0000 Subject: [PATCH 12/19] Dedupe historical JS seeds by path (collapse cache-buster variants) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live testing surfaced 22 needs-review items that were really ~2 tokens: web archives store one file under many cache-buster query strings (api_data.js?v=1596328976710, ?v=1596328981612, …), and the historical seed layer fed all 11 variants as separate scan targets, so the same secrets were re-found in each. HistoricalResult.js_urls() now dedupes by (host, path), keeping one URL per unique JS file. Fewer redundant fetches, and a finding count that reflects distinct files instead of cache-bust noise. (The 22 were also all "AI validation skipped" — the CLI run lacked GEMINI_API_KEY, so every structural match correctly routed to manual review rather than being dropped; and the tokens are apiDoc documentation examples the AI triages when enabled. This dedupe fixes the multiplier.) Test: 11 api_data.js?v=… + api_project.js collapse to 2 seeds. Suite 247 -> 248, ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- backend/historical.py | 19 +++++++++++++++++-- backend/tests/test_historical.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/backend/historical.py b/backend/historical.py index 93da67a..995452d 100644 --- a/backend/historical.py +++ b/backend/historical.py @@ -74,8 +74,23 @@ def paths(self) -> list[str]: def js_urls(self) -> list[str]: """Discovered URLs that look like JavaScript — the highest-value scan - seeds, since bundled JS is where secrets most often leak.""" - return [u for u in self.urls if urlparse(u).path.lower().endswith(".js")] + seeds, since bundled JS is where secrets most often leak. + + Deduplicated by (host, path): archives store the same file under many + cache-buster query strings (app.js?v=1, app.js?v=2, …); scanning each + variant re-finds the same secrets, so we keep only one URL per unique + file. This is what collapses e.g. 11 api_data.js?v=… variants to one.""" + seen: set[tuple[str, str]] = set() + out: list[str] = [] + for u in self.urls: + p = urlparse(u) + if not p.path.lower().endswith(".js"): + continue + key = ((p.hostname or "").lower(), p.path) + if key not in seen: + seen.add(key) + out.append(u) + return out def to_dict(self) -> dict: return { diff --git a/backend/tests/test_historical.py b/backend/tests/test_historical.py index 759df18..004d9e2 100644 --- a/backend/tests/test_historical.py +++ b/backend/tests/test_historical.py @@ -51,6 +51,20 @@ def test_paths_and_js_urls(self): assert r.paths == ["/a", "/b", "/static/app.js"] assert r.js_urls() == ["https://example.com/static/app.js"] + def test_js_urls_dedupes_cache_buster_variants(self): + # The same file under many ?v=… cache-busters must collapse to one seed — + # this is what turns 11 api_data.js?v=… into a single fetch. + r = historical.HistoricalResult(domain="example.com", urls=[ + "https://rest.example.com/docs/api_data.js?v=1596328976710", + "https://rest.example.com/docs/api_data.js?v=1596328981612", + "https://rest.example.com/docs/api_data.js?v=1743672225794", + "https://rest.example.com/docs/api_project.js?v=1", + ]) + assert r.js_urls() == [ + "https://rest.example.com/docs/api_data.js?v=1596328976710", + "https://rest.example.com/docs/api_project.js?v=1", + ] + class _Resp: def __init__(self, status_code: int, text: str = ""): From b1dcae7087774387932817e17cee951b597d225b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 20:16:58 +0000 Subject: [PATCH 13/19] Add dashboard domain-mode + deep-scan API (deep-ASM slice 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole deep-ASM pipeline is now drivable from the web UI, not just the CLI. Backend: • POST /api/deep-scans — runs orchestrator.run_deep_scan as a streaming background task (mirrors the single-scan endpoint): returns a scan_id, per-host progress flows over the existing /ws/logs/{scan_id} socket. DeepScanRequest validates + caps inputs. • run_deep_scan gained a broadcast hook: it emits enumerate / historical / probe / per-host / deep_scan_complete events and forwards the socket to each host's run_scan, so the live terminal shows real progress. • Report endpoint serves the combined multi-target report (generate_deep_scan_html) for results flagged deep_scan. Frontend: • A DEEP toggle turns the target box into a whole-domain scan (bare domain in → enumerate + historical + probe + scan every live host). • The WS handler tracks deep mode and finalises on deep_scan_complete instead of the first per-host scan_complete; a new case reports the aggregate totals. Tests: API route registered, auth required, input caps, scan starts and returns a scan_id (orchestrator + save mocked, offline). JS syntax-checked. Suite 248 -> 252, ruff clean. Passive posture intact. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- CHANGELOG.md | 8 +++ backend/data/secretnode.db-journal | Bin 0 -> 16928 bytes backend/main.py | 95 +++++++++++++++++++++++++++- backend/orchestrator.py | 26 +++++++- backend/tests/test_deep_scan_api.py | 66 +++++++++++++++++++ frontend/index.html | 63 +++++++++++++++--- 6 files changed, 244 insertions(+), 14 deletions(-) create mode 100644 backend/data/secretnode.db-journal create mode 100644 backend/tests/test_deep_scan_api.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bcaff2..507b248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ All notable changes to SecretNode are documented here. This project adheres to ## [Unreleased] ### Added +- **Dashboard domain-mode + deep-scan API (deep-ASM slice 6).** The whole deep-ASM pipeline is now + drivable from the web UI, not just the CLI. New `POST /api/deep-scans` runs a domain-wide deep scan + as a streaming background task — per-host progress flows over the existing `/ws/logs/{scan_id}` + WebSocket (`run_deep_scan` gained a `broadcast` hook and emits enumerate/probe/per-host/complete + events), and the report endpoint serves the combined multi-target report for deep results. + Frontend: a **DEEP toggle** turns the target box into a whole-domain scan (bare domain in → + enumerate + historical + probe + scan-all), finalising on `deep_scan_complete` rather than + per-host. API tests added (route, auth, input caps, start). Passive; authorized-scope only. - **Historical bundles fed into the scan (deep-ASM slice 3.5).** `run_scan()` gains a `seed_urls` parameter — externally-supplied asset URLs are fetched and scanned alongside the live crawl, deduped against it (capped by `MAX_SEED_URLS`). `run_deep_scan(include_historical=True)` now diff --git a/backend/data/secretnode.db-journal b/backend/data/secretnode.db-journal new file mode 100644 index 0000000000000000000000000000000000000000..f615335fa558ea5f3898f91da8ac8cfc876ce625 GIT binary patch literal 16928 zcmeI&ze>YE9Ki9*onj)$HH%eT+)9Vom=FVvD&_ImAMsV^yd;(v_BHgodq*DBv%;PU6fkKAYb{c=JqD*+=AU{c71`kD+$O9ESwKG-UXKT?1 z7z_QS$D6-Uz7PQf5I_I{1Q0*~0R#|0009K%0!?9Roj_w z6!4P#ws=hQvg!lKJ+nNiFNA(K1Azbn2q1s}0tg_000IagfB*vP1+JxT#YdKH+pan6 uop7ZJuicklRp_eM@Ac0cAF~+M{I None: } +class DeepScanRequest(BaseModel): + """Domain-wide deep scan: enumerate → probe → scan each live host.""" + domain: str + crawl_pages: int = DEFAULT_CRAWL_PAGES + verify: bool = False + only_verified: bool = False + include_historical: bool = False + max_targets: int = orchestrator.MAX_TARGETS + + @field_validator("domain") + @classmethod + def validate_domain(cls, v: str) -> str: + v = v.strip() + if not v or len(v) > 253: + raise ValueError("Provide a domain (or host/URL) up to 253 characters.") + return v + + @field_validator("crawl_pages") + @classmethod + def _cap_crawl(cls, v: int) -> int: + return max(1, min(v, MAX_CRAWL_PAGES_CAP)) + + @field_validator("max_targets") + @classmethod + def _cap_targets(cls, v: int) -> int: + return max(1, min(v, 100)) + + +@app.post("/api/deep-scans", status_code=202, dependencies=[Depends(require_api_key)]) +async def start_deep_scan(request: DeepScanRequest, http_request: Request) -> dict[str, Any]: + """Start a background domain-wide deep scan (enumerate → probe → scan each + live host). Returns a scan_id; connect to /ws/logs/{scan_id} to stream + progress. Discovery is passive; authorized-scope use only.""" + client_ip = http_request.client.host if http_request.client else "unknown" + logger.info("AUDIT deep_scan_request client=%s domain=%s", client_ip, request.domain) + + active_count = sum(1 for e in _registry.values() if not e["task"].done()) + if active_count >= MAX_CONCURRENT_SCANS: + raise HTTPException( + status_code=429, + detail=f"{active_count} scan(s) already running (limit: {MAX_CONCURRENT_SCANS}).", + ) + + scan_id = str(uuid.uuid4()) + + async def broadcaster(event: dict[str, Any]) -> None: + await manager.broadcast_scan(scan_id, event) + + async def _run() -> None: + try: + deep = await orchestrator.run_deep_scan( + request.domain, + max_crawl_pages=request.crawl_pages, + verify=request.verify, + only_verified=request.only_verified, + include_historical=request.include_historical, + max_targets=request.max_targets, + broadcast=broadcaster, + ) + result = deep.to_dict() + result["scan_id"] = scan_id + result["target_url"] = deep.domain + result["deep_scan"] = True + totals = result.get("totals", {}) + result["status"] = ("complete" + if (totals.get("confirmed") or totals.get("needs_review") + or totals.get("posture_issues")) else "clean") + _registry[scan_id]["meta"] = result + await save_scan(scan_id, result) + except asyncio.CancelledError: + await manager.broadcast_scan(scan_id, {"type": "scan_cancelled", "scan_id": scan_id}) + except Exception as exc: + logger.exception("Deep scan %s crashed", scan_id) + await manager.broadcast_scan(scan_id, { + "type": "scan_error", "scan_id": scan_id, "error": str(exc)}) + + task = asyncio.create_task(_run(), name=f"deep-scan-{scan_id}") + _registry[scan_id] = { + "state": ScanState(), + "task": task, + "meta": {"scan_id": scan_id, "target_url": request.domain, + "status": "running", "deep_scan": True}, + } + logger.info("Deep scan %s started for %s", scan_id, request.domain) + return {"scan_id": scan_id, "domain": request.domain, "status": "started", + "ws_url": f"/ws/logs/{scan_id}", + "message": "Connect to ws_url to receive live events."} + + @app.post("/api/scans/{scan_id}/stop", status_code=200, dependencies=[Depends(require_api_key)]) async def stop_scan(scan_id: str) -> dict[str, Any]: """Signal a running scan to stop cooperatively.""" @@ -481,7 +571,10 @@ async def get_scan_report( ) if format == "html": - body = report_gen.generate_html_report(scan, agency_name=agency_name) + # A domain-wide deep scan aggregates many hosts — render the combined + # multi-target report instead of the single-target one. + body = (report_gen.generate_deep_scan_html(scan) if scan.get("deep_scan") + else report_gen.generate_html_report(scan, agency_name=agency_name)) return HTMLResponse(content=body, headers={ "Content-Disposition": f'inline; filename="secretnode_report_{scan_id[:8]}.html"' }) diff --git a/backend/orchestrator.py b/backend/orchestrator.py index 42da1fc..f08c5c6 100644 --- a/backend/orchestrator.py +++ b/backend/orchestrator.py @@ -196,6 +196,7 @@ async def run_deep_scan( only_verified: bool = False, max_targets: int = MAX_TARGETS, include_historical: bool = False, + broadcast: Callable[[dict], Awaitable[None]] | None = None, enumerate_fn: EnumerateFn = recon.enumerate_subdomains, scan_fn: ScanFn = scanner.run_scan, client_factory: ClientFactory = scanner.build_client, @@ -214,18 +215,28 @@ async def run_deep_scan( url = target if "://" in target else f"https://{host}" result = DeepScanResult(domain=host) scan = await scan_fn(target_url=url, max_crawl_pages=max_crawl_pages, - verify=verify, only_verified=only_verified) + verify=verify, only_verified=only_verified, broadcast=broadcast) result.live_hosts = [url] result.hosts = [_summarise_scan(host, url, scan)] result.scans = [scan] return result + async def emit(event: dict) -> None: + if broadcast: + await broadcast(event) + + def log(msg: str, level: str = "INFO") -> dict: + return {"type": "log", "level": level, "message": msg} + original_host = recon._host_of(target) result = DeepScanResult(domain=domain) async with client_factory() as client: + await emit(log(f"Deep scan of {domain} — enumerating subdomains (Certificate Transparency)…")) enum = await enumerate_fn(client, domain) result.subdomains = enum.subdomains result.enum_sources = enum.sources + await emit(log(f"Enumerated {len(enum.subdomains)} subdomain(s)" + + (f" via {', '.join(enum.sources)}" if enum.sources else ""))) # Optional: recover historical URLs (Wayback/CommonCrawl) once for the # domain. They enrich BOTH host discovery (hostnames seen in the archive) @@ -234,11 +245,14 @@ async def run_deep_scan( js_by_host: dict[str, list[str]] = {} hist_hosts: list[str] = [] if include_historical: + await emit(log("Recovering historical URLs from public archives (Wayback/CommonCrawl)…")) hist = await discover_historical_fn(client, domain) result.historical_urls = hist.count hist_hosts = [recon._host_of(u) for u in hist.urls] for u in hist.js_urls(): js_by_host.setdefault(recon._host_of(u), []).append(u) + await emit(log(f"Recovered {hist.count} historical URL(s); " + f"{sum(len(v) for v in js_by_host.values())} archived JS seed(s)")) # Candidate hosts, in-scope and deduped. The host the caller actually # typed is ALWAYS included first — enumeration must never be able to drop @@ -265,19 +279,24 @@ async def run_deep_scan( error="skipped: resolves to a private/internal address (SSRF guard)")) # 'unresolved' hosts are silently dropped — nothing to scan. + await emit(log(f"Probing {len(safe_hosts)} candidate host(s) for liveness…")) result.live_hosts = await probe_live_hosts(client, safe_hosts) if not result.live_hosts: result.error = enum.error or "no live hosts found" + await emit(log("No live hosts to scan.", "WARN")) + await emit({"type": "deep_scan_complete", "totals": result.to_dict()["totals"]}) return result targets = result.live_hosts[:max(1, max_targets)] - for url in targets: + await emit(log(f"{len(result.live_hosts)} host(s) live — scanning {len(targets)}")) + for i, url in enumerate(targets, 1): host = recon._host_of(url) + await emit(log(f"[host {i}/{len(targets)}] scanning {url}")) try: scan = await scan_fn(target_url=url, max_crawl_pages=max_crawl_pages, verify=verify, only_verified=only_verified, - seed_urls=js_by_host.get(host, [])) + seed_urls=js_by_host.get(host, []), broadcast=broadcast) except Exception as exc: # a single host must not sink the whole run logger.debug("deep scan: host %s failed: %s", url, exc) result.hosts.append(HostScan(host=host, url=url, @@ -286,4 +305,5 @@ async def run_deep_scan( result.scans.append(scan) result.hosts.append(_summarise_scan(host, url, scan)) + await emit({"type": "deep_scan_complete", "totals": result.to_dict()["totals"]}) return result diff --git a/backend/tests/test_deep_scan_api.py b/backend/tests/test_deep_scan_api.py new file mode 100644 index 0000000..3841df6 --- /dev/null +++ b/backend/tests/test_deep_scan_api.py @@ -0,0 +1,66 @@ +"""API-level tests for the domain deep-scan endpoint (deep-ASM slice 6). + +Follows the same setup as the other API tests: main.py refuses to import without +SECRETNODE_API_KEY, so a shared default is set (via setdefault, so all test +modules agree) BEFORE importing main, and main is imported once. The orchestrator +is mocked so nothing touches the network.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("SECRETNODE_API_KEY", "test-key-for-pytest") + +import pytest +from fastapi.testclient import TestClient + +import main # noqa: E402 (must follow the env setup above) + +HEADERS = {"X-API-Key": os.environ["SECRETNODE_API_KEY"]} + + +@pytest.fixture +def client(): + return TestClient(main.app) + + +def test_deep_scan_route_registered(): + paths = {route.path for route in main.app.routes} + assert "/api/deep-scans" in paths + + +def test_deep_scan_requires_api_key(client): + with client: + r = client.post("/api/deep-scans", json={"domain": "example.com"}) + assert r.status_code == 401 + + +def test_deep_scan_request_caps_inputs(): + req = main.DeepScanRequest(domain="example.com", crawl_pages=9999, max_targets=9999) + assert req.crawl_pages <= main.MAX_CRAWL_PAGES_CAP + assert req.max_targets == 100 + with pytest.raises(ValueError): + main.DeepScanRequest(domain=" ") + + +def test_deep_scan_starts_and_returns_scan_id(client, monkeypatch): + import orchestrator + + async def fake_deep(domain, **_kw): + return orchestrator.DeepScanResult(domain=domain) + + async def _noop_save(*_a, **_k): + return None + + monkeypatch.setattr(main.orchestrator, "run_deep_scan", fake_deep) + monkeypatch.setattr(main, "save_scan", _noop_save) # avoid post-teardown DB write race + with client: + r = client.post( + "/api/deep-scans", + headers=HEADERS, + json={"domain": "example.com", "include_historical": True}, + ) + assert r.status_code == 202 + body = r.json() + assert body["scan_id"] and body["ws_url"].endswith(body["scan_id"]) + assert body["domain"] == "example.com" diff --git a/frontend/index.html b/frontend/index.html index 9d7d348..e1ec0db 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -508,6 +508,10 @@ VERIFY + @@ -1169,6 +1173,12 @@ break; case 'scan_complete': + // In a deep scan this fires once PER HOST — don't finalise the UI or + // disconnect; just note the host finished. deep_scan_complete ends it. + if (state.deepScan) { + appendLog(`Host scan complete (${state.allFindings.length} confirmed so far)`, 'SYS'); + break; + } setScanningUI(false); setProgress(100, 'complete'); state.totalScans++; @@ -1188,6 +1198,21 @@ }, 2000); break; + case 'deep_scan_complete': { + setScanningUI(false); + setProgress(100, 'complete'); + state.totalScans++; + animateNumber('stat-scans', state.totalScans); + const t = msg.totals || {}; + appendLog(`=== Deep scan complete — ${t.hosts_scanned || 0} host(s), ` + + `${t.confirmed || 0} confirmed, ${t.needs_review || 0} needs-review, ` + + `${t.posture_issues || 0} posture ===`, 'SYS'); + toast(`Deep scan complete — ${t.hosts_scanned || 0} host(s), ${t.confirmed || 0} confirmed`, + (t.confirmed ? 'error' : 'success'), 8000); + setTimeout(() => { if (state.ws) state.ws.close(); }, 2000); + break; + } + case 'scan_cancelled': setScanningUI(false); setProgress(0, 'cancelled'); @@ -1205,10 +1230,18 @@ } // ── Start Scan ──────────────────────────────────────────────────────────── + // When DEEP is toggled, the input takes a bare domain rather than a full URL. + function onDeepToggle() { + const deep = $('deep-input')?.checked; + $('target-input').placeholder = deep ? 'example.com (whole-domain deep scan)' : 'https://example.com'; + } + window.onDeepToggle = onDeepToggle; + async function startScan() { - const url = $('target-input').value.trim(); - if (!url) { toast('Please enter a target URL', 'warn'); return; } - if (!url.startsWith('http')) { toast('URL must begin with http:// or https://', 'warn'); return; } + const target = $('target-input').value.trim(); + const deep = $('deep-input')?.checked || false; + if (!target) { toast(deep ? 'Please enter a domain' : 'Please enter a target URL', 'warn'); return; } + if (!deep && !target.startsWith('http')) { toast('URL must begin with http:// or https://', 'warn'); return; } clearFindings(); clearAssets(); @@ -1216,19 +1249,29 @@ setProgress(0, ''); $('stage-text').textContent = ''; - appendLog(`Starting scan for: ${url}`, 'INFO'); + appendLog(deep ? `Starting DEEP scan for domain: ${target}` : `Starting scan for: ${target}`, 'INFO'); + state.deepScan = deep; // a deep scan finalises on deep_scan_complete, not per-host state.wsReconnected = false; // allow one auto-reconnect for this scan setScanningUI(true); try { - const resp = await fetch('/api/scans', { + const endpoint = deep ? '/api/deep-scans' : '/api/scans'; + const payload = deep + ? { + domain: target, + crawl_pages: parseInt($('crawl-pages-input').value, 10) || 1, + verify: $('verify-input')?.checked || false, + include_historical: true, + } + : { + target_url: target, + crawl_pages: parseInt($('crawl-pages-input').value, 10) || 1, + verify: $('verify-input')?.checked || false, + }; + const resp = await fetch(endpoint, { method: 'POST', headers: apiHeaders({ 'Content-Type': 'application/json' }), - body: JSON.stringify({ - target_url: url, - crawl_pages: parseInt($('crawl-pages-input').value, 10) || 1, - verify: $('verify-input')?.checked || false, - }), + body: JSON.stringify(payload), }); if (resp.status === 401) { From 32994b5b7c25cd14d2ca8a2dba58a6b7d320ecb3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 20:24:37 +0000 Subject: [PATCH 14/19] Add surface intelligence: endpoint extraction + associated-host graph (slices 5 & 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend/surface.py mines already-fetched assets (passive — no new target requests beyond the deeper-crawl fetches) for two things: Slice 5 — extract_endpoints(): URLs/paths referenced in JS/HTML (fetch()/axios targets, /api/… routes) resolved to absolute URLs. A live crawl never links to these; run_scan now fetches the same-site .js endpoints ONE LEVEL DEEPER so code-referenced bundles get secret-scanned too. Slice 4 — extract_referenced_hosts(): the external hosts an asset talks to (CDNs, APIs, third parties), aggregated across a scan into the associated-asset graph (third-party / connected-infra surface). Wiring: • run_scan returns discovered_endpoints + associated_hosts; the surface step runs after asset collection, bounded by MAX_ENDPOINT_SEEDS / MAX_DISCOVERED_ENDPOINTS, behind EXTRACT_SURFACE. • DeepScanResult aggregates associated_hosts across hosts. • Single-target and deep reports gain an "Attack Surface Intelligence" section (associated hosts + endpoints referenced in code). All extractor regexes are bounded (no nested quantifiers) — verified ReDoS-safe on pathological input (0.006s). 8 surface tests + a run_scan integration test (endpoints/hosts populated, deeper .js fetched). Suite 252 -> 260, ruff clean. Passive posture intact. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V6XK23ZKmYtCovRte9a73E --- .env.example | 8 +++ CHANGELOG.md | 9 +++ backend/orchestrator.py | 2 + backend/report.py | 22 +++++++ backend/scanner.py | 48 ++++++++++++++ backend/surface.py | 114 ++++++++++++++++++++++++++++++++++ backend/tests/test_scanner.py | 44 +++++++++++++ backend/tests/test_surface.py | 66 ++++++++++++++++++++ 8 files changed, 313 insertions(+) create mode 100644 backend/surface.py create mode 100644 backend/tests/test_surface.py diff --git a/.env.example b/.env.example index 13e9f32..e5f3268 100644 --- a/.env.example +++ b/.env.example @@ -93,6 +93,14 @@ MAX_HISTORICAL_URLS=2000 # Cap on externally-supplied seed assets fetched per scan (e.g. historical JS # bundles fed into a --deep-scan --with-historical run). MAX_SEED_URLS=200 +# ── Surface intelligence (endpoints referenced in code + associated hosts) ── +# Mine fetched JS/HTML for referenced endpoints and external hosts, and fetch +# same-site .js endpoints one level deeper. Set false to disable. +EXTRACT_SURFACE=true +# Same-site .js endpoints (referenced in code) to fetch for the deeper crawl. +MAX_ENDPOINT_SEEDS=50 +# Cap on how many discovered endpoints to store/report. +MAX_DISCOVERED_ENDPOINTS=300 # Skip JS assets larger than this many bytes (default 5 MB). MAX_ASSET_BYTES=5242880 # Only surface AI-confirmed findings at or above this confidence (0-100). diff --git a/CHANGELOG.md b/CHANGELOG.md index 507b248..ecfe2e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ All notable changes to SecretNode are documented here. This project adheres to ## [Unreleased] ### Added +- **Surface intelligence: endpoints + associated-host graph (deep-ASM slices 5 & 4).** New + `backend/surface.py` mines every fetched asset (passively, no new target requests) for two things: + **(5)** URLs/paths referenced in the JavaScript — `fetch()`/`axios` targets, `/api/…` routes a live + page crawl never links to — and then fetches same-site `.js` endpoints **one level deeper** so + code-referenced bundles get secret-scanned too; and **(4)** the external hosts each asset talks to + (CDNs, APIs, third parties), aggregated into an **associated-asset graph**. `run_scan` now returns + `discovered_endpoints` + `associated_hosts`; both reports gain an "Attack Surface Intelligence" + section. Extractor regexes are bounded/ReDoS-safe. Config: `EXTRACT_SURFACE`, `MAX_ENDPOINT_SEEDS`, + `MAX_DISCOVERED_ENDPOINTS`. - **Dashboard domain-mode + deep-scan API (deep-ASM slice 6).** The whole deep-ASM pipeline is now drivable from the web UI, not just the CLI. New `POST /api/deep-scans` runs a domain-wide deep scan as a streaming background task — per-host progress flows over the existing `/ws/logs/{scan_id}` diff --git a/backend/orchestrator.py b/backend/orchestrator.py index f08c5c6..2de922f 100644 --- a/backend/orchestrator.py +++ b/backend/orchestrator.py @@ -118,6 +118,8 @@ def to_dict(self) -> dict: "historical_urls": self.historical_urls, "confirmed_findings": self._aggregate("confirmed_findings"), "needs_review_findings": self._aggregate("needs_review_findings"), + "associated_hosts": sorted({h for s in self.scans + for h in s.get("associated_hosts", [])}), "totals": { "subdomains": len(self.subdomains), "live_hosts": len(self.live_hosts), diff --git a/backend/report.py b/backend/report.py index f1aadb1..89d2c26 100644 --- a/backend/report.py +++ b/backend/report.py @@ -226,6 +226,17 @@ def _posture_row(p: dict[str, Any]) -> str: posture_html = "\n".join(_posture_row(p) for p in sorted(posture, key=_sort_key)) or \ 'No header/misconfiguration issues detected.' + # Attack-surface intelligence (slices 5 & 4): endpoints referenced in code and + # the external hosts this asset talks to. Capped for readability. + endpoints = scan.get("discovered_endpoints", []) or [] + assoc_hosts = scan.get("associated_hosts", []) or [] + endpoints_html = "".join( + f'
  • {html.escape(e)}
    • ' for e in endpoints[:80] + ) or '
  • None discovered.
    • ' + if len(endpoints) > 80: + endpoints_html += f'
  • … and {len(endpoints) - 80} more.
    • ' + hosts_html = ", ".join(html.escape(h) for h in assoc_hosts) or "—" + return f""" @@ -355,6 +366,13 @@ def _posture_row(p: dict[str, Any]) -> str: {review_html} +

    Attack Surface Intelligence

    +
    Associated hosts (external services this + asset references — CDNs, APIs, third parties): {hosts_html}
    +
    Endpoints referenced in code ({len(endpoints)} discovered — URLs/paths + the JavaScript calls that a page crawl would miss):
    +
      {endpoints_html}
    +