Add service attribution and destination intelligence

.env.example

@@ -54,6 +54,13 @@ NETBOT_LIVE_RING_DEFAULT_QUERY_LIMIT=250
 NETBOT_LIVE_RING_MAX_QUERY_LIMIT=2000
 NETBOT_LIVE_RING_TTL_SECONDS=0
 
+# Local, metadata-only destination attribution. No external lookup is performed.
+NETBOT_SERVICE_ATTRIBUTION_ENABLED=true
+NETBOT_SERVICE_ATTRIBUTION_REGISTRY=backend/app/data/service_fingerprints.json
+NETBOT_SERVICE_ATTRIBUTION_DNS_WINDOW_SEC=300
+NETBOT_SERVICE_ATTRIBUTION_MAX_REASONS=8
+NETBOT_SERVICE_ATTRIBUTION_UNKNOWN_RATE_WARN=0.75
+
 # Bounded batch persistence. Counters and health are exposed in Ops Snapshot.
 NETBOT_PERSISTENCE_BATCH_ENABLED=true
 NETBOT_PERSISTENCE_PACKET_BATCH_SIZE=500

README.md

@@ -66,6 +66,7 @@ or PCAP artifacts.
 | Conversation Timeline | Active | Protocol, alert, destination, and lifecycle events are correlated per flow. | Timeline summaries and metadata pass through central redaction. |
 | Flow Risk Scoring | Active | Explainable `0..100` scoring covers alerts, volume, DNS failures, unusual protocols/ports, and destinations. | Risk is an investigation aid, not an automated verdict. |
 | Offline PCAP Flow Summary | Active | Offline analysis returns flows, conversations, protocol summaries, timelines, and risk distribution. | Existing API response fields remain compatible; output stays redacted. |
+| Service Attribution / Destination Intelligence | Active MVP | DNS, visible TLS SNI, HTTP Host, recent DNS-answer correlation, ASN metadata, and a local fingerprint registry produce confidence-scored destination labels with evidence. Browser/container identity and CDN infrastructure are handled conservatively; encrypted traffic without evidence remains Unknown. | Local metadata-only inference; no external lookup, TLS decryption, browser history, cookies, credentials, or payload inspection. |
 | Deep Packet Inspection | Active MVP | Inspect renders a searchable layer tree, safe bytes view, streams, and Expert Info. | No TLS decryption; visible metadata and ASCII previews are centrally redacted. |
 | Display Filters | Active MVP | Safe packet and flow filter parser covers text, equality, range, and boolean operators. | Filters run on redacted metadata and never use Python `eval`. |
 | Offline PCAP Deep Analysis | Active MVP | Offline results include packet details, Expert Info, and stream summaries. | Previous API fields remain compatible; raw secrets are not exposed. |
@@ -256,6 +257,7 @@ and surfaced in Ops Snapshot metrics.
 - [Performance Pipeline](docs/PERFORMANCE_PIPELINE.md)
 - [Performance Validation](docs/PERFORMANCE_VALIDATION.md)
 - [Flow Analysis And Protocol Intelligence](docs/FLOW_ANALYSIS.md)
+- [Service Attribution And Destination Intelligence](docs/SERVICE_ATTRIBUTION.md)
 - [Deep Packet Inspection](docs/DEEP_PACKET_INSPECTION.md)
 - [Agent Operational QA Checklist](docs/AGENT_QA_CHECKLIST.md)
 - [Deployment Overview](docs/DEPLOYMENT_OVERVIEW.md)
@@ -484,9 +486,9 @@ powershell -ExecutionPolicy Bypass -File .\packaging\windows\build.ps1
 
 - Complete the performance foundation with a flow-aware worker pool, live ring
   buffer, and benchmark/soak validation.
-- Add conservative Service Attribution / Destination Intelligence using DNS,
-  TLS SNI, HTTP Host, QUIC-visible metadata, ASN, and local fingerprints. Low
-  confidence remains `Unknown`; no TLS decryption or credential collection.
+- Validate and tune the implemented metadata-only Service Attribution registry
+  against authorized deployment traffic while preserving conservative Unknown
+  results for weak or hidden evidence.
 - Build a read-only Incident / Correlation Engine after attribution quality is
   validated, then consider a strictly read-only AI Analyst.
 - Versioned SQLite schema migrations and longer-lived deployment operations.

backend/app/data/service_fingerprints.json

@@ -0,0 +1,28 @@
+{
+  "version": 1,
+  "services": [
+    {"service_name":"YouTube","category":"Video Streaming","domain_patterns":["youtube.com","*.youtube.com","googlevideo.com","*.googlevideo.com","ytimg.com","*.ytimg.com","youtu.be"],"sni_patterns":["youtube.com","*.youtube.com","googlevideo.com","*.googlevideo.com","ytimg.com","*.ytimg.com"],"http_host_patterns":["youtube.com","*.youtube.com","googlevideo.com","*.googlevideo.com","ytimg.com","*.ytimg.com"],"asn_org_patterns":["google"],"cdn":true,"risk_notes":"Known video streaming service; review only for policy or volume anomalies."},
+    {"service_name":"Netflix","category":"Video Streaming","domain_patterns":["netflix.com","*.netflix.com","nflxvideo.net","*.nflxvideo.net","nflximg.net","*.nflximg.net"],"asn_org_patterns":["netflix"],"cdn":true},
+    {"service_name":"Twitch","category":"Video Streaming","domain_patterns":["twitch.tv","*.twitch.tv","ttvnw.net","*.ttvnw.net"],"asn_org_patterns":["twitch"],"cdn":true},
+    {"service_name":"Telegram Web","category":"Messaging","domain_patterns":["web.telegram.org","telegram.org","*.telegram.org","t.me","*.t.me"],"asn_org_patterns":["telegram"]},
+    {"service_name":"WhatsApp Web","category":"Messaging","domain_patterns":["web.whatsapp.com","whatsapp.com","*.whatsapp.com","whatsapp.net","*.whatsapp.net"],"asn_org_patterns":["meta platforms","facebook"]},
+    {"service_name":"Discord","category":"Messaging","domain_patterns":["discord.com","*.discord.com","discord.gg","*.discord.gg","discordapp.com","*.discordapp.com","discordapp.net","*.discordapp.net"],"asn_org_patterns":["discord"],"cdn":true},
+    {"service_name":"GitHub","category":"Developer Platform","domain_patterns":["github.com","*.github.com","githubusercontent.com","*.githubusercontent.com","githubassets.com","*.githubassets.com"],"asn_org_patterns":["github"]},
+    {"service_name":"GitLab","category":"Developer Platform","domain_patterns":["gitlab.com","*.gitlab.com"],"asn_org_patterns":["gitlab"]},
+    {"service_name":"Docker Hub","category":"Developer Platform","domain_patterns":["docker.com","*.docker.com","docker.io","*.docker.io"],"asn_org_patterns":["docker"]},
+    {"service_name":"npm Registry","category":"Developer Platform","domain_patterns":["npmjs.com","*.npmjs.com","npmjs.org","*.npmjs.org"],"asn_org_patterns":["npm"]},
+    {"service_name":"PyPI","category":"Developer Platform","domain_patterns":["pypi.org","*.pypi.org","pythonhosted.org","*.pythonhosted.org"],"asn_org_patterns":["python software foundation"]},
+    {"service_name":"Google","category":"Search / Cloud","domain_patterns":["google.com","*.google.com","gstatic.com","*.gstatic.com","1e100.net","*.1e100.net"],"asn_org_patterns":["google"]},
+    {"service_name":"Microsoft","category":"Search / Cloud","domain_patterns":["microsoft.com","*.microsoft.com","live.com","*.live.com","office.com","*.office.com"],"asn_org_patterns":["microsoft"]},
+    {"service_name":"AWS","category":"Cloud","domain_patterns":["amazonaws.com","*.amazonaws.com","aws.amazon.com"],"asn_org_patterns":["amazon","aws"],"cdn":true},
+    {"service_name":"Azure","category":"Cloud","domain_patterns":["azure.com","*.azure.com","azurewebsites.net","*.azurewebsites.net"],"asn_org_patterns":["microsoft"]},
+    {"service_name":"Instagram","category":"Social","domain_patterns":["instagram.com","*.instagram.com","cdninstagram.com","*.cdninstagram.com"],"asn_org_patterns":["meta platforms","facebook"],"cdn":true},
+    {"service_name":"X / Twitter","category":"Social","domain_patterns":["x.com","*.x.com","twitter.com","*.twitter.com","twimg.com","*.twimg.com"],"asn_org_patterns":["twitter"]},
+    {"service_name":"TikTok","category":"Social","domain_patterns":["tiktok.com","*.tiktok.com","tiktokcdn.com","*.tiktokcdn.com","byteoversea.com","*.byteoversea.com"],"asn_org_patterns":["bytedance"],"cdn":true},
+    {"service_name":"Cloudflare CDN","category":"CDN","domain_patterns":["cloudflare.com","*.cloudflare.com","cloudflare.net","*.cloudflare.net"],"asn_org_patterns":["cloudflare"],"cdn":true},
+    {"service_name":"Akamai","category":"CDN","domain_patterns":["akamaized.net","*.akamaized.net","akamai.net","*.akamai.net","akamaiedge.net","*.akamaiedge.net"],"asn_org_patterns":["akamai"],"cdn":true},
+    {"service_name":"Fastly","category":"CDN","domain_patterns":["fastly.net","*.fastly.net","fastlylb.net","*.fastlylb.net"],"asn_org_patterns":["fastly"],"cdn":true},
+    {"service_name":"Google CDN","category":"CDN","domain_patterns":["googleusercontent.com","*.googleusercontent.com","ggpht.com","*.ggpht.com"],"asn_org_patterns":["google"],"cdn":true},
+    {"service_name":"Amazon CloudFront","category":"CDN","domain_patterns":["cloudfront.net","*.cloudfront.net"],"asn_org_patterns":["amazon"],"cdn":true}
+  ]
+}

backend/app/main.py

@@ -119,6 +119,7 @@ def _observability_snapshot() -> dict[str, Any]:
         "packet_queue": sniffer_service.packet_queue_stats(),
         "flow_worker_pool": sniffer_service.flow_worker_pool_stats(),
         "live_ring_buffer": sniffer_service.live_ring_buffer_stats(),
+        "service_attribution": sniffer_service.service_attribution_stats(),
         "persistence": sniffer_service.persistence_stats(),
         "auto_block": sniffer_service.auto_block_stats(),
     }

backend/app/services/monitoring_service.py

@@ -187,6 +187,7 @@ def build_monitoring_metrics(
     packet_queue = dict(observability.get("packet_queue") or {})
     flow_worker_pool = dict(observability.get("flow_worker_pool") or {})
     live_ring_buffer = dict(observability.get("live_ring_buffer") or {})
+    service_attribution = dict(observability.get("service_attribution") or {})
     event_aggregator = dict(observability.get("event_aggregator") or {})
     websocket = dict(observability.get("websocket") or {})
     persistence = dict(observability.get("persistence") or {})
@@ -288,6 +289,18 @@ def build_monitoring_metrics(
     pressure_reasons.extend(
         reason for reason in live_ring_reasons if reason not in pressure_reasons
     )
+    service_attribution_reasons = [
+        reason
+        for value in service_attribution.get("pressure_reasons") or []
+        if (reason := str(value)).startswith("service_attribution_")
+        and len(reason) <= 80
+        and reason.replace("_", "").isalnum()
+    ]
+    pressure_reasons.extend(
+        reason
+        for reason in service_attribution_reasons
+        if reason not in pressure_reasons
+    )
     pressure_reasons.extend(
         reason for reason in persistence_reasons if reason not in pressure_reasons
     )
@@ -343,6 +356,7 @@ def build_monitoring_metrics(
         or websocket_drops >= _int(event_aggregator.get("client_queue_max") or 1000)
         or flow_worker_pool.get("health") == "critical"
         or live_ring_buffer.get("health") == "critical"
+        or service_attribution.get("health") == "critical"
     ):
         health = "critical"
 
@@ -588,6 +602,41 @@ def build_monitoring_metrics(
             },
             "pressure_reasons": live_ring_reasons,
         },
+        "service_attribution": {
+            "enabled": bool(service_attribution.get("enabled", True)),
+            "health": str(service_attribution.get("health") or "healthy"),
+            "registry_size": _int(service_attribution.get("registry_size")),
+            "attributed_flows_total": _int(
+                service_attribution.get("attributed_flows_total")
+            ),
+            "unknown_flows_total": _int(service_attribution.get("unknown_flows_total")),
+            "high_confidence_total": _int(
+                service_attribution.get("high_confidence_total")
+            ),
+            "medium_confidence_total": _int(
+                service_attribution.get("medium_confidence_total")
+            ),
+            "low_confidence_total": _int(
+                service_attribution.get("low_confidence_total")
+            ),
+            "encrypted_unknown_total": _int(
+                service_attribution.get("encrypted_unknown_total")
+            ),
+            "cdn_only_total": _int(service_attribution.get("cdn_only_total")),
+            "attribution_errors_total": _int(
+                service_attribution.get("attribution_errors_total")
+            ),
+            "avg_attribution_latency_ms": _number(
+                service_attribution.get("avg_attribution_latency_ms")
+            ),
+            "p95_attribution_latency_ms": _number(
+                service_attribution.get("p95_attribution_latency_ms")
+            ),
+            "last_error": _safe_persistence_error(
+                service_attribution.get("last_error")
+            ),
+            "pressure_reasons": service_attribution_reasons,
+        },
         "persistence": {
             "enabled": bool(
                 persistence.get("persistence_enabled", persistence.get("enabled"))