ML stages

index.html

@@ -98,16 +98,19 @@ <h2>Linux Kernel Ring 0 Visualization</h2>
     <script src="/static/js/memory-card.js?v=5"></script>
     <script src="/static/js/waits-card.js?v=3"></script>
     <script src="/static/js/sockets-card.js?v=1"></script>
+    <script src="/static/js/flow-card.js?v=4"></script>
     <script src="/static/js/ipc-ui.js?v=8"></script>
     <script src="/static/js/flow-ui.js?v=8"></script>
-    <script src="/static/js/isolation-ui.js?v=11"></script>
+    <script src="/static/js/namespace-card.js?v=2"></script>
+    <script src="/static/js/isolation-ui.js?v=15"></script>
     <script src="/static/js/process-files-ui.js?v=1"></script>
     <script src="/static/js/ui-chrome.js?v=2"></script>
-    <script src="/static/js/active-connections.js?v=5"></script>
+    <script src="/static/js/active-connections.js?v=6"></script>
     <script src="/static/js/nginx_files.js?v=6"></script>
     <script src="/static/js/right-semicircle-menu.js?v=53"></script>
-    <script src="/static/js/kernel-dna.js?v=49"></script>
-    <script src="/static/js/network-stack.js?v=61"></script>
+    <script src="/static/js/kernel-dna.js?v=50"></script>
+    <script src="/static/js/network-stack.js?v=66"></script>
+    <script src="/static/js/network-socket-body.js?v=2"></script>
     <script src="/static/js/devices-belt.js?v=25"></script>
     <script src="/static/js/aes-ref.js?v=3"></script>
     <script src="/static/js/crypto-belt.js?v=61"></script>
@@ -123,7 +126,7 @@ <h2>Linux Kernel Ring 0 Visualization</h2>
             console.error('❌ RightSemicircleMenuManager class NOT loaded!');
         }
     </script>
-    <script src="/static/js/main.js?v=234"></script>
+    <script src="/static/js/main.js?v=236"></script>
     <script src="/static/js/kernel-tape.js?v=13"></script>
 </body>
 </html>

kernel_ai/http/api_handlers/network_system.py

@@ -32,7 +32,9 @@ def _payload():
 def network_stack_realtime():
     return api_json(
         lambda: _network_service.get_network_stack_realtime(
-            network_stack_prev=get_state_container(current_app).network_stack_prev
+            network_stack_prev=get_state_container(current_app).network_stack_prev,
+            prefer_local=request.args.get("local"),
+            prefer_remote=request.args.get("remote"),
         )
     )
 

kernel_ai/ml/http_features.py

@@ -0,0 +1,152 @@
+"""Per-IP windows over HTTP events. No raw body and no home paths leave here."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from urllib.parse import unquote_plus
+
+from kernel_ai.ml.http_parse import HttpEvent
+from kernel_ai.ml.http_rules import (
+    _CMDI,
+    _JNDI,
+    _LFI,
+    _SQLI,
+    _XSS,
+    _UNUSUAL_METHOD,
+    classify_window,
+)
+
+WINDOW_SEC = 60
+
+# Fixed order shared by train and the worker.
+HTTP_FEATURE_ORDER = [
+    "count",
+    "rate_per_sec",
+    "uniq_paths",
+    "mean_path_len",
+    "max_path_len",
+    "frac_404",
+    "frac_4xx",
+    "frac_5xx",
+    "frac_2xx",
+    "has_dotfile",
+    "has_traversal",
+    "has_sqli",
+    "has_xss",
+    "has_cmdi",
+    "has_jndi",
+    "unusual_method",
+    "mean_query_len",
+]
+
+_DOTFILE = (".env", ".git", ".svn", ".htaccess", ".htpasswd", ".aws")
+
+
+@dataclass
+class HttpWindow:
+    ts: float
+    src_ip: str
+    window_sec: int
+    events: list[HttpEvent] = field(default_factory=list)
+    features: dict[str, float] = field(default_factory=dict)
+    label: str = "benign"
+    cls: str = "benign"
+    why: str = ""
+
+    def vector(self) -> list[float]:
+        return [float(self.features.get(name, 0.0)) for name in HTTP_FEATURE_ORDER]
+
+
+def _flags(events: list[HttpEvent]) -> dict[str, float]:
+    has_dot = 0.0
+    has_trav = 0.0
+    has_sqli = 0.0
+    has_xss = 0.0
+    has_cmdi = 0.0
+    has_jndi = 0.0
+    unusual = 0.0
+    for event in events:
+        hay = f"{event.path} {unquote_plus(event.query or '')} {event.body}"
+        path_l = event.path.lower()
+        if any(token in path_l for token in _DOTFILE):
+            has_dot = 1.0
+        if _LFI.search(hay):
+            has_trav = 1.0
+        if _SQLI.search(hay):
+            has_sqli = 1.0
+        if _XSS.search(hay):
+            has_xss = 1.0
+        if _CMDI.search(hay):
+            has_cmdi = 1.0
+        if _JNDI.search(hay):
+            has_jndi = 1.0
+        if event.method in _UNUSUAL_METHOD:
+            unusual = 1.0
+    return {
+        "has_dotfile": has_dot,
+        "has_traversal": has_trav,
+        "has_sqli": has_sqli,
+        "has_xss": has_xss,
+        "has_cmdi": has_cmdi,
+        "has_jndi": has_jndi,
+        "unusual_method": unusual,
+    }
+
+
+def features_of(events: list[HttpEvent], *, window_sec: int = WINDOW_SEC) -> dict[str, float]:
+    n = len(events)
+    if n == 0:
+        return {name: 0.0 for name in HTTP_FEATURE_ORDER}
+    paths = [event.path for event in events]
+    lens = [len(event.path) for event in events]
+    qlens = [len(event.query) for event in events]
+    n404 = sum(1 for event in events if event.status == 404)
+    n4xx = sum(1 for event in events if 400 <= event.status < 500)
+    n5xx = sum(1 for event in events if event.status >= 500)
+    n2xx = sum(1 for event in events if 200 <= event.status < 300)
+    flags = _flags(events)
+    out = {
+        "count": float(n),
+        "rate_per_sec": n / max(1.0, float(window_sec)),
+        "uniq_paths": float(len(set(paths))),
+        "mean_path_len": sum(lens) / n,
+        "max_path_len": float(max(lens)),
+        "frac_404": n404 / n,
+        "frac_4xx": n4xx / n,
+        "frac_5xx": n5xx / n,
+        "frac_2xx": n2xx / n,
+        "mean_query_len": sum(qlens) / n,
+    }
+    out.update(flags)
+    return out
+
+
+def build_windows(
+    events: list[HttpEvent],
+    *,
+    window_sec: int = WINDOW_SEC,
+    label: bool = True,
+) -> list[HttpWindow]:
+    """Bucket events by src_ip and floor(ts / window). Empty IP is dropped."""
+    buckets: dict[tuple[str, int], list[HttpEvent]] = defaultdict(list)
+    for event in events:
+        if not event.src_ip:
+            continue
+        slot = int(event.ts // window_sec)
+        buckets[(event.src_ip, slot)].append(event)
+
+    windows: list[HttpWindow] = []
+    for (src_ip, slot), rows in sorted(buckets.items(), key=lambda item: item[0][1]):
+        feats = features_of(rows, window_sec=window_sec)
+        win = HttpWindow(
+            ts=float(slot * window_sec),
+            src_ip=src_ip,
+            window_sec=window_sec,
+            events=rows,
+            features=feats,
+        )
+        if label:
+            win.label, win.cls, win.why = classify_window(rows, feats)
+        windows.append(win)
+    return windows

kernel_ai/ml/http_gap.py

@@ -0,0 +1,76 @@
+"""Stage 2 report: HTTP/SIEM attempts vs kernel mutations (hole and forest noise)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from typing import Any
+
+from kernel_ai.ml.config import MLConfig
+from kernel_ai.ml.http_join import JOIN_SEC, is_web_kernel_anomaly
+from kernel_ai.ml.store import fetch_anomalies_since, fetch_http_labels
+
+
+def _ts(row: dict[str, Any]) -> float:
+    try:
+        return float(row.get("ts") or 0.0)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def report_gap(
+    attempts: list[dict[str, Any]],
+    kernel_anomalies: list[dict[str, Any]],
+    *,
+    slack_sec: float = JOIN_SEC,
+) -> dict:
+    """attempts = labeled HTTP attempt windows; kernel = Stage 1/2/4/5 rows."""
+    kernel = [row for row in kernel_anomalies if not str(row.get("source") or "").startswith("stage9")]
+    hits = 0
+    missed = 0
+    for attempt in attempts:
+        ats = _ts(attempt)
+        near = any(ats <= _ts(row) <= ats + slack_sec for row in kernel)
+        if near:
+            hits += 1
+        else:
+            missed += 1
+    lonely = 0
+    lonely_web = 0
+    for row in kernel:
+        kts = _ts(row)
+        if any(abs(kts - _ts(attempt)) <= slack_sec for attempt in attempts):
+            continue
+        lonely += 1
+        if is_web_kernel_anomaly(row):
+            lonely_web += 1
+    return {
+        "attempts": len(attempts),
+        "kernel_anomalies": len(kernel),
+        "attempt_with_kernel": hits,
+        "attempt_without_kernel": missed,
+        "kernel_without_attempt": lonely,
+        "kernel_web_without_attempt": lonely_web,
+        "slack_sec": slack_sec,
+        "note": (
+            "attempt_without_kernel is the hole (wordlist, SIEM saw it, forest silent). "
+            "kernel_without_attempt is host-rhythm noise (dashboard polls, retrain)."
+        ),
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="HTTP attempts vs kernel mutations")
+    parser.add_argument("--hours", type=int, default=24)
+    parser.add_argument("--slack-sec", type=float, default=JOIN_SEC)
+    args = parser.parse_args(argv)
+    cfg = MLConfig()
+    labels = [r for r in fetch_http_labels(cfg.dsn, hours=args.hours) if r.get("label") == "attempt"]
+    kernel = fetch_anomalies_since(cfg.dsn, hours=args.hours)
+    payload = report_gap(labels, kernel, slack_sec=args.slack_sec)
+    print(json.dumps(payload, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

kernel_ai/ml/http_join.py

@@ -0,0 +1,98 @@
+"""Join an HTTP attempt to a later web-stack kernel mutation → success."""
+
+from __future__ import annotations
+
+from typing import Any
+
+WEB_COMMS = {"nginx", "gunicorn", "python", "python3"}
+KERNEL_SOURCES = {
+    "stage4_sequence",
+    "stage5_process",
+    "stage8_markov",
+    "stage8_lstm",
+}
+JOIN_SEC = 30.0
+SUCCESS_FEATURE = "http_success:exec"
+SUCCESS_POSITION = 0.12
+
+
+def _comm(anomaly: dict[str, Any]) -> str:
+    meta = anomaly.get("meta") if isinstance(anomaly.get("meta"), dict) else {}
+    return str(meta.get("comm") or meta.get("parent_comm") or "").lower()
+
+
+def is_web_kernel_anomaly(anomaly: dict[str, Any]) -> bool:
+    comm = _comm(anomaly)
+    if comm in WEB_COMMS:
+        return True
+    parent = ""
+    meta = anomaly.get("meta") if isinstance(anomaly.get("meta"), dict) else {}
+    parent = str(meta.get("parent_comm") or "").lower()
+    if parent in WEB_COMMS:
+        return True
+    feature = str(anomaly.get("feature") or "")
+    return any(name in feature for name in WEB_COMMS)
+
+
+def join_success(
+    attempts: list[dict[str, Any]],
+    kernel_anomalies: list[dict[str, Any]],
+    *,
+    slack_sec: float = JOIN_SEC,
+) -> list[dict[str, Any]]:
+    """Each web-stack kernel hit that follows an attempt becomes one success row."""
+    if not attempts or not kernel_anomalies:
+        return []
+    out: list[dict[str, Any]] = []
+    seen: set[tuple[str, int]] = set()
+    for kernel in kernel_anomalies:
+        if not is_web_kernel_anomaly(kernel):
+            continue
+        kts = float(kernel.get("ts") or 0.0)
+        matched = None
+        for attempt in attempts:
+            ats = float(attempt.get("ts") or 0.0)
+            if ats <= kts <= ats + slack_sec:
+                matched = attempt
+                break
+        if matched is None:
+            continue
+        key = (str(matched.get("src_ip") or ""), int(kts))
+        if key in seen:
+            continue
+        seen.add(key)
+        cls = str(matched.get("cls") or "anomaly")
+        src_ip = str(matched.get("src_ip") or "")
+        ip_tail = ".".join(src_ip.split(".")[-2:]) if src_ip else "--"
+        meta = dict(kernel.get("meta") or {})
+        meta.update(
+            {
+                "stage": 9,
+                "kind": "success",
+                "cls": cls,
+                "src_ip": src_ip,
+                "why": f"attempt:{cls} then {kernel.get('source')}",
+                "kernel_source": kernel.get("source"),
+            }
+        )
+        out.append(
+            {
+                "source": "stage9_success",
+                "feature": SUCCESS_FEATURE,
+                "subsystem": "sched",
+                "type": SUCCESS_FEATURE,
+                "severity": "high",
+                "score": 1.0,
+                "value": slack_sec,
+                "baseline_mean": None,
+                "baseline_std": None,
+                "position": SUCCESS_POSITION,
+                "message": (
+                    f"HTTP attempt ({cls}) then web-stack kernel mutation "
+                    f"({kernel.get('source')}, ip …{ip_tail})"
+                ),
+                "meta": meta,
+                "ts": kts,
+            }
+        )
+    return out