From 474af5457343bca714b9407e3fa9b48f2533b240 Mon Sep 17 00:00:00 2001 From: Aleksei Fedorov Date: Sun, 16 Aug 2026 13:28:48 +0000 Subject: [PATCH 1/2] ML stages --- kernel_ai/ml/config.py | 44 ++++- kernel_ai/ml/polygon.py | 21 ++- kernel_ai/ml/retrain.py | 20 +++ kernel_ai/ml/store.py | 146 ++++++++++++++- kernel_ai/ml/worker.py | 168 +++++++++++++++++- kernel_ai/services/system_view.py | 77 +++++++- kernel_ai/services/telemetry_orchestration.py | 26 ++- 7 files changed, 478 insertions(+), 24 deletions(-) diff --git a/kernel_ai/ml/config.py b/kernel_ai/ml/config.py index 0929e0c..b6b3937 100644 --- a/kernel_ai/ml/config.py +++ b/kernel_ai/ml/config.py @@ -55,9 +55,10 @@ class MLConfig: # settle so we don't fire on the cold-start transient). warmup_samples: int = _env_int("KERNEL_AI_ML_WARMUP", 30) - # Robust z-score thresholds. A feature whose current value sits this many - # standard deviations above its baseline becomes a mutation. - z_warn: float = _env_float("KERNEL_AI_ML_Z_WARN", 4.0) + # Robust z-score thresholds. Calibrated on live ring-0.sh traffic + # (2026-08-15): z=4 fired on dashboard poll bursts (ctxt/pgfault/retrans). + # 6σ keeps the real spikes (z 7–12) and drops the 4.3–5.5 band. + z_warn: float = _env_float("KERNEL_AI_ML_Z_WARN", 6.0) z_crit: float = _env_float("KERNEL_AI_ML_Z_CRIT", 7.0) # Persist the raw feature snapshot each tick (useful for later training / @@ -94,8 +95,14 @@ class MLConfig: if_contamination: float = _env_float("KERNEL_AI_ML_IF_CONTAMINATION", 0.02) if_n_estimators: int = _env_int("KERNEL_AI_ML_IF_TREES", 200) # Min seconds between IsolationForest mutations (avoid per-tick spam during a - # sustained anomaly). - if_cooldown_sec: float = _env_float("KERNEL_AI_ML_IF_COOLDOWN_SEC", 15.0) + # sustained anomaly). 15s capped at 240/h and the live host sat on that + # ceiling; 45s caps Stage 2 at 80/h. + if_cooldown_sec: float = _env_float("KERNEL_AI_ML_IF_COOLDOWN_SEC", 45.0) + # IsolationForest predict() still flags ~2% by construction, but on this + # host the decision boundary is noisy (flag rate 0.20–0.53 at retrain). + # Require a minimum -decision_function before emit; live weak hits were + # 0.05–0.09, real-looking ones 0.11+. + if_min_score: float = _env_float("KERNEL_AI_ML_IF_MIN_SCORE", 0.10) # --- Stage 3 (drift + auto-retrain) --- # Window of recent snapshots used to measure drift (minutes). @@ -203,6 +210,33 @@ class MLConfig: # above be derived from the live distribution of a given host. stage8_log_every_sec: float = _env_float("KERNEL_AI_ML_STAGE8_LOG_EVERY_SEC", 60.0) + # --- Stage 9 (HTTP attempt model + success join) --- + # Default OFF: do not emit on PROD until offline metrics exist. + enable_stage9: bool = os.getenv("KERNEL_AI_ML_STAGE9", "false").lower() == "true" + http_nginx_log: str = os.getenv( + "KERNEL_AI_ML_HTTP_NGINX_LOG", + "/var/log/nginx/kernel-ai.json.log", + ) + http_app_log: str = os.getenv("KERNEL_AI_ML_HTTP_APP_LOG", "") + http_model_path: str = os.getenv( + "KERNEL_AI_ML_HTTP_MODEL_PATH", + str(_DATA_DIR / "http_attempt_latest.joblib"), + ) + http_model_prev_path: str = os.getenv( + "KERNEL_AI_ML_HTTP_MODEL_PREV", + str(_DATA_DIR / "http_attempt_prev.joblib"), + ) + http_window_sec: int = _env_int("KERNEL_AI_ML_HTTP_WINDOW_SEC", 60) + http_cooldown_sec: float = _env_float("KERNEL_AI_ML_HTTP_COOLDOWN_SEC", 45.0) + http_min_score: float = _env_float("KERNEL_AI_ML_HTTP_MIN_SCORE", 0.55) + http_min_precision: float = _env_float("KERNEL_AI_ML_HTTP_MIN_PRECISION", 0.60) + http_max_flag_rate: float = _env_float("KERNEL_AI_ML_HTTP_MAX_FLAG", 0.45) + http_join_sec: float = _env_float("KERNEL_AI_ML_HTTP_JOIN_SEC", 30.0) + http_mlflow_experiment: str = os.getenv( + "KERNEL_AI_ML_HTTP_EXPERIMENT", "kernel_dna_http_attempt" + ) + http_mlflow_model: str = os.getenv("KERNEL_AI_ML_HTTP_MODEL", "kernel_dna_http_attempt") + @property def alpha(self) -> float: return 2.0 / (max(2, self.baseline_window) + 1.0) diff --git a/kernel_ai/ml/polygon.py b/kernel_ai/ml/polygon.py index b93720a..de9a69a 100644 --- a/kernel_ai/ml/polygon.py +++ b/kernel_ai/ml/polygon.py @@ -379,7 +379,10 @@ def run_mimicry(*, stide_warn: float = 0.30, markov_warn: float = 2.5) -> dict: def main(argv: list[str] | None = None) -> int: logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") parser = argparse.ArgumentParser(description="Stage 5/7/8 local polygon (dev only)") - parser.add_argument("mode", choices=("dry-run", "live", "list", "mimicry", "stream")) + parser.add_argument( + "mode", + choices=("dry-run", "live", "list", "mimicry", "stream", "http-wordlist", "http-rce"), + ) parser.add_argument( "--scenario", action="append", @@ -398,8 +401,24 @@ def main(argv: list[str] | None = None) -> int: print(f"{name:16} {meta['technique']:8} [{live}] {meta['description']}") print(f"{'mimicry':16} {'T1106':8} [dry-only] STIDE miss / Markov hit (Stage 8)") print(f"{'stream':16} {'L2':8} [dry-only] Stage 6 socket e2e (demo collector)") + print(f"{'http-wordlist':16} {'HTTP':8} [dry-only] Stage 9 scanner window") + print(f"{'http-rce':16} {'HTTP':8} [dry-only] Stage 9 attempt then exec join") return 0 + if args.mode == "http-wordlist": + from kernel_ai.ml.http_polygon import run_http_wordlist + + result = run_http_wordlist() + print(result) + return 0 if result["pass"] else 1 + + if args.mode == "http-rce": + from kernel_ai.ml.http_polygon import run_http_rce + + result = run_http_rce() + print(result) + return 0 if result["pass"] else 1 + if args.mode == "mimicry": result = run_mimicry() return 0 if result["pass"] else 1 diff --git a/kernel_ai/ml/retrain.py b/kernel_ai/ml/retrain.py index 58efd27..99b1b21 100644 --- a/kernel_ai/ml/retrain.py +++ b/kernel_ai/ml/retrain.py @@ -20,6 +20,7 @@ import argparse import logging +import os from kernel_ai.ml.config import MLConfig from kernel_ai.ml.drift import compute_drift @@ -78,6 +79,23 @@ def _refresh_markov(cfg: MLConfig, metrics: dict) -> None: logger.warning("Stage 8 Markov build failed: %s", exc) +def _refresh_http(cfg: MLConfig, metrics: dict) -> None: + """Best-effort Stage 9 retrain. Guardrails live in http_train (keep previous).""" + if os.getenv("KERNEL_AI_ML_HTTP_RETRAIN", "true").lower() != "true": + return + try: + from kernel_ai.ml.http_train import train as train_http + + http_metrics = train_http(cfg, min_samples=40, persist=True) + metrics["http_precision"] = float(http_metrics.get("holdout_precision") or 0) + metrics["http_flag_rate"] = float(http_metrics.get("holdout_flag_rate") or 0) + logger.info("HTTP attempt model refreshed: %s", http_metrics) + except SystemExit as exc: + logger.info("HTTP retrain skipped: %s", exc) + except Exception as exc: # noqa: BLE001 + logger.warning("HTTP retrain failed: %s", exc) + + def run(*, only_if_drift: bool, min_samples: int, stide_only: bool = False) -> int: cfg = MLConfig() metrics: dict = {} @@ -85,6 +103,7 @@ def run(*, only_if_drift: bool, min_samples: int, stide_only: bool = False) -> i if stide_only: _refresh_stide(cfg, metrics) _refresh_markov(cfg, metrics) + _refresh_http(cfg, metrics) logger.info("stide-only retrain complete: %s", metrics) return 0 @@ -114,6 +133,7 @@ def run(*, only_if_drift: bool, min_samples: int, stide_only: bool = False) -> i _refresh_stide(cfg, metrics) _refresh_markov(cfg, metrics) + _refresh_http(cfg, metrics) logger.info("retrain complete: %s", metrics) return 0 diff --git a/kernel_ai/ml/store.py b/kernel_ai/ml/store.py index d899011..8b1cb16 100644 --- a/kernel_ai/ml/store.py +++ b/kernel_ai/ml/store.py @@ -89,11 +89,52 @@ last_seen timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (parent_comm, child_comm) ); + +CREATE TABLE IF NOT EXISTS ml_http_labels ( + id bigserial PRIMARY KEY, + ts timestamptz NOT NULL, + src_ip text NOT NULL, + window_sec integer NOT NULL DEFAULT 60, + label text NOT NULL, + cls text, + why text, + teacher text NOT NULL DEFAULT 'rules', + features jsonb +); +CREATE INDEX IF NOT EXISTS ml_http_labels_ts_idx ON ml_http_labels (ts); + +CREATE TABLE IF NOT EXISTS ml_http_windows ( + ts timestamptz NOT NULL DEFAULT now(), + src_ip text NOT NULL, + window_sec integer NOT NULL DEFAULT 60, + features jsonb NOT NULL, + label text, + cls text +); +CREATE INDEX IF NOT EXISTS ml_http_windows_ts_idx ON ml_http_windows (ts); """ +# libpq's default client cert is ~/.postgresql/postgresql.crt. Gunicorn runs +# with ProtectHome=yes, so opening that path returns EACCES and the whole +# connection dies — the DNA /ml-anomalies API then looks empty even though +# the worker is writing. A missing file is fine; an unreadable HOME is not. +# Point the cert paths at a location that simply does not exist. +_NO_CLIENT_CERT = "/nonexistent/kernel-ai-pg.crt" +_NO_CLIENT_KEY = "/nonexistent/kernel-ai-pg.key" + + +def connect_kwargs(*, autocommit: bool = True) -> dict: + """psycopg kwargs that do not touch the service user's home directory.""" + return { + "autocommit": autocommit, + "sslcert": _NO_CLIENT_CERT, + "sslkey": _NO_CLIENT_KEY, + } + + def connect(dsn: str, *, autocommit: bool = True) -> psycopg.Connection: - return psycopg.connect(dsn, autocommit=autocommit) + return psycopg.connect(dsn, **connect_kwargs(autocommit=autocommit)) def ensure_schema(conn: psycopg.Connection) -> None: @@ -245,6 +286,65 @@ def prune(self, features_hours: int, anomalies_hours: int) -> None: "DELETE FROM ml_proc_snapshots WHERE ts < now() - make_interval(hours => %s)", (features_hours,), ) + cur.execute( + "DELETE FROM ml_http_labels WHERE ts < now() - make_interval(hours => %s)", + (anomalies_hours,), + ) + cur.execute( + "DELETE FROM ml_http_windows WHERE ts < now() - make_interval(hours => %s)", + (features_hours,), + ) + + def insert_http_labels(self, rows: list[dict]) -> int: + if not rows: + return 0 + with self.conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO ml_http_labels + (ts, src_ip, window_sec, label, cls, why, teacher, features) + VALUES + (to_timestamp(%(ts)s), %(src_ip)s, %(window_sec)s, %(label)s, + %(cls)s, %(why)s, %(teacher)s, %(features)s) + """, + [ + { + "ts": float(r["ts"]), + "src_ip": r["src_ip"], + "window_sec": int(r.get("window_sec") or 60), + "label": r["label"], + "cls": r.get("cls"), + "why": r.get("why"), + "teacher": r.get("teacher") or "rules", + "features": Json(r.get("features") or {}), + } + for r in rows + ], + ) + return len(rows) + + def insert_http_windows(self, rows: list[dict]) -> None: + if not rows: + return + with self.conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO ml_http_windows (ts, src_ip, window_sec, features, label, cls) + VALUES (to_timestamp(%(ts)s), %(src_ip)s, %(window_sec)s, %(features)s, + %(label)s, %(cls)s) + """, + [ + { + "ts": float(r["ts"]), + "src_ip": r["src_ip"], + "window_sec": int(r.get("window_sec") or 60), + "features": Json(r.get("features") or {}), + "label": r.get("label"), + "cls": r.get("cls"), + } + for r in rows + ], + ) def close(self) -> None: try: @@ -394,3 +494,47 @@ def fetch_recent_anomalies(dsn: str, *, since_seconds: int = 120, limit: int = 1 except Exception as exc: # noqa: BLE001 - API must degrade gracefully logger.warning("fetch_recent_anomalies failed: %s", exc) return [] + + +def fetch_http_labels(dsn: str, *, hours: int = 24, limit: int = 20000) -> list[dict]: + """Labeled HTTP windows for training / gap reports. Empty on DB trouble.""" + try: + with connect(dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT extract(epoch from ts) AS ts, src_ip, window_sec, label, cls, + why, teacher, features + FROM ml_http_labels + WHERE ts > now() - make_interval(hours => %s) + ORDER BY ts DESC + LIMIT %s + """, + (max(1, int(hours)), limit), + ) + cols = [d.name for d in cur.description] + return [dict(zip(cols, row)) for row in cur.fetchall()] + except Exception as exc: # noqa: BLE001 + logger.warning("fetch_http_labels failed: %s", exc) + return [] + + +def fetch_anomalies_since(dsn: str, *, hours: int = 24, limit: int = 5000) -> list[dict]: + """Anomalies with epoch ts for offline joins. Empty on DB trouble.""" + try: + with connect(dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT extract(epoch from ts) AS ts, source, feature, type, severity, + score, message, meta + FROM ml_anomalies + WHERE ts > now() - make_interval(hours => %s) + ORDER BY ts DESC + LIMIT %s + """, + (max(1, int(hours)), limit), + ) + cols = [d.name for d in cur.description] + return [dict(zip(cols, row)) for row in cur.fetchall()] + except Exception as exc: # noqa: BLE001 + logger.warning("fetch_anomalies_since failed: %s", exc) + return [] diff --git a/kernel_ai/ml/worker.py b/kernel_ai/ml/worker.py index d1c7190..ae48e37 100644 --- a/kernel_ai/ml/worker.py +++ b/kernel_ai/ml/worker.py @@ -224,6 +224,26 @@ def __init__(self, cfg: MLConfig | None = None) -> None: self.deep_scorer.ready, ) + # Stage 9 — HTTP attempts (supervised) + success join. Off by default. + self.http_model = None + self._http_model_mtime: float | None = None + self._http_tails: list = [] + self._http_events: list = [] + self._http_attempts: list[dict] = [] + self._last_http_emit: dict[str, float] = {} + if self.cfg.enable_stage9: + from kernel_ai.ml.http_tail import HttpLogTail + + for path in (self.cfg.http_nginx_log, self.cfg.http_app_log): + if path: + self._http_tails.append(HttpLogTail(path)) + self._maybe_load_http_model() + logger.info( + "Stage 9 enabled: tails=%d model=%s", + len(self._http_tails), + bool(self.http_model), + ) + def _maybe_load_model(self) -> None: """Load / hot-reload the IsolationForest artifact if present and changed. @@ -248,6 +268,25 @@ def _maybe_load_model(self) -> None: except Exception as exc: # noqa: BLE001 - keep running on Stage 1 only logger.warning("failed to load Stage 2 model: %s", exc) + def _maybe_load_http_model(self) -> None: + if not self.cfg.enable_stage9: + return + path = self.cfg.http_model_path + try: + mtime = os.path.getmtime(path) + except OSError: + return + if self._http_model_mtime is not None and mtime <= self._http_model_mtime: + return + try: + from kernel_ai.ml.http_model import HttpAttemptModel + + self.http_model = HttpAttemptModel.load(path) + self._http_model_mtime = mtime + logger.info("loaded Stage 9 HTTP model: %s (%s)", path, self.http_model.meta) + except Exception as exc: # noqa: BLE001 + logger.warning("failed to load Stage 9 HTTP model: %s", exc) + def _maybe_load_seq_model(self) -> None: """Load / hot-reload the STIDE profile artifact if present and changed.""" if not self.cfg.enable_stage4: @@ -456,6 +495,104 @@ def _tick_stage8(self) -> dict | None: self._last_stage8_emit = now return self.deep_scorer.build_anomaly(score, self.cfg) + def _tick_http(self) -> list[dict]: + """Score recent IP windows; remember attempts for the success join.""" + if not self.cfg.enable_stage9: + return [] + now = time.time() + fresh = [] + for tail in self._http_tails: + fresh.extend(tail.read_new(now=now)) + keep_after = now - max(self.cfg.http_window_sec * 2, 120) + self._http_events = [e for e in self._http_events if e.ts >= keep_after] + self._http_events.extend(fresh) + self._http_attempts = [a for a in self._http_attempts if now - float(a.get("ts") or 0) <= 120] + if self.http_model is None or not self._http_events: + return [] + + from kernel_ai.ml.http_features import build_windows + from kernel_ai.ml.http_rules import named_attempt_class + + positions = { + "scanner": 0.22, + "flood": 0.24, + "method": 0.26, + "lfi": 0.28, + "sensitive": 0.30, + "sqli": 0.32, + "xss": 0.34, + "cmdi": 0.36, + "jndi": 0.38, + } + out: list[dict] = [] + for win in build_windows(self._http_events, window_sec=self.cfg.http_window_sec, label=True): + is_attempt, proba = self.http_model.predict_one(win.features) + if not is_attempt or proba < self.cfg.http_min_score: + continue + cls = named_attempt_class(win.label, win.cls) + if not cls: + continue + feature = f"http_attempt:{cls}" + last = self._last_http_emit.get(feature, 0.0) + if (now - last) < self.cfg.http_cooldown_sec: + continue + self._last_http_emit[feature] = now + ip_tail = ".".join(win.src_ip.split(".")[-2:]) if win.src_ip else "--" + rec = { + "source": "stage9_http", + "feature": feature, + "subsystem": "net", + "type": feature, + "severity": "high" if cls in {"sqli", "cmdi", "jndi", "lfi"} else "medium", + "score": round(proba, 4), + "value": float(win.features.get("count") or 0), + "baseline_mean": None, + "baseline_std": None, + "position": positions.get(cls, 0.25), + "message": ( + f"HTTP attempt {cls} (p={proba:.2f}, {int(win.features.get('count') or 0)} req, " + f"ip …{ip_tail}; {win.why})" + ), + "meta": { + "stage": 9, + "kind": "attempt", + "cls": cls, + "src_ip": win.src_ip, + "why": win.why, + "proba": round(proba, 4), + }, + "ts": win.ts, + } + out.append(rec) + self._http_attempts.append(rec) + return out + + def _join_http_success(self, kernel_anomalies: list[dict]) -> list[dict]: + if not self.cfg.enable_stage9 or not self._http_attempts or not kernel_anomalies: + return [] + from kernel_ai.ml.http_join import join_success + + now = time.time() + enriched = [] + for row in kernel_anomalies: + meta = dict(row.get("meta") or {}) + pid = meta.get("pid") + if pid and not meta.get("comm"): + try: + with open(f"/proc/{int(pid)}/comm", "r", encoding="utf-8") as fh: + meta["comm"] = fh.read().strip() + except (OSError, ValueError): + pass + item = dict(row) + item["meta"] = meta + item.setdefault("ts", now) + enriched.append(item) + return join_success( + self._http_attempts, + enriched, + slack_sec=self.cfg.http_join_sec, + ) + def _tick_process(self) -> list[dict]: """Stage 5: sample processes, score lineage/baselines, persist whitelist.""" if self.proc_extractor is None or self.proc_detector is None: @@ -514,7 +651,11 @@ def _tick(self) -> int: is_anom, if_score = False, 0.0 logger.warning("isoforest scoring failed: %s", exc) now = time.time() - if is_anom and (now - self._last_if_emit) >= self.cfg.if_cooldown_sec: + if ( + is_anom + and if_score >= self.cfg.if_min_score + and (now - self._last_if_emit) >= self.cfg.if_cooldown_sec + ): anomalies.append(_build_isoforest_anomaly(if_score, scores, self.cfg)) self._last_if_emit = now @@ -543,15 +684,31 @@ def _tick(self) -> int: except Exception as exc: # noqa: BLE001 logger.warning("stage8 scoring failed: %s", exc) + # Stage 9: HTTP attempts, then join to web-stack kernel mutations. + if self.cfg.enable_stage9: + try: + http_anoms = self._tick_http() + anomalies.extend(http_anoms) + except Exception as exc: # noqa: BLE001 + logger.warning("http scoring failed: %s", exc) + try: + anomalies.extend(self._join_http_success(anomalies)) + except Exception as exc: # noqa: BLE001 + logger.warning("http success join failed: %s", exc) + # Stage 7: ATT&CK / Sigma-lite labels on whatever Stages 1–5 emitted. if self.cfg.enable_stage7 and anomalies: try: from kernel_ai.ml.attribution import enrich_anomalies - anomalies = enrich_anomalies( - anomalies, - min_confidence=self.cfg.attack_min_confidence, - ) + http_rows = [a for a in anomalies if str(a.get("source") or "").startswith("stage9")] + core_rows = [a for a in anomalies if not str(a.get("source") or "").startswith("stage9")] + if core_rows: + core_rows = enrich_anomalies( + core_rows, + min_confidence=self.cfg.attack_min_confidence, + ) + anomalies = core_rows + http_rows except Exception as exc: # noqa: BLE001 logger.warning("attribution enrich failed: %s", exc) @@ -590,6 +747,7 @@ def run(self) -> None: # Pick up a freshly retrained model without a restart. self._maybe_load_model() self._maybe_load_seq_model() + self._maybe_load_http_model() if self.deep_scorer is not None: self.deep_scorer.maybe_reload() except Exception as exc: # noqa: BLE001 - keep the loop alive diff --git a/kernel_ai/services/system_view.py b/kernel_ai/services/system_view.py index 64556e9..3a28bdb 100644 --- a/kernel_ai/services/system_view.py +++ b/kernel_ai/services/system_view.py @@ -896,7 +896,47 @@ def _read_cgroup_v2_stats(cgroup_path): } -def get_isolation_context(): +_ISOLATION_SNAPSHOT = os.environ.get("ISOLATION_OUT", "/run/kernel-ai/isolation.json") +_ISOLATION_MAX_AGE = 8.0 +_ISOLATION_SAMPLE_RANK = { + "nginx": 0, + "gunicorn": 1, + "sshd": 2, + "systemd": 3, + "filebeat": 4, + "dbus-daemon": 5, +} + + +def _read_isolation_snapshot(): + """The root collector's machine-wide namespace table, if present and fresh.""" + try: + with open(_ISOLATION_SNAPSHOT, "r", encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + return None + ts = data.get("ts") + try: + age = time.time() - float(ts) + except (TypeError, ValueError): + return None + if age > _ISOLATION_MAX_AGE: + return None + data["source"] = "collector" + data["timestamp"] = datetime.now().isoformat() + return data + + +def _rank_isolation_samples(names): + unique = [] + for name in names: + if name and name not in unique: + unique.append(name) + unique.sort(key=lambda name: (_ISOLATION_SAMPLE_RANK.get(name, 80), name)) + return unique[:5] + + +def _scan_isolation_context(): namespace_keys = ["mnt", "pid", "net", "ipc", "uts", "user"] namespace_labels = {"mnt": "MNT", "pid": "PID", "net": "NET", "ipc": "IPC", "uts": "UTS", "user": "USER"} namespace_counts = {k: {} for k in namespace_keys} @@ -904,6 +944,7 @@ def get_isolation_context(): namespace_samples = {k: {} for k in namespace_keys} cgroup_aggregates = {} total_scanned = 0 + host_inodes = {ns_name: read_namespace_inode(1, ns_name) for ns_name in namespace_keys} for proc in psutil.process_iter(["pid", "name", "memory_info"]): try: @@ -929,7 +970,7 @@ def get_isolation_context(): ns_map = namespace_counts[ns_name] ns_map[inode] = ns_map.get(inode, 0) + 1 samples = namespace_samples[ns_name].setdefault(inode, []) - if len(samples) < 5 and proc_name not in samples: + if proc_name not in samples: samples.append(proc_name) except (psutil.NoSuchProcess, psutil.AccessDenied, KeyError): continue @@ -938,17 +979,21 @@ def get_isolation_context(): for ns_name in namespace_keys: entries = namespace_counts[ns_name] unique_count = len(entries) - dominant_inode = None - dominant_count = 0 - if entries: + host_inode = host_inodes.get(ns_name) + if host_inode and host_inode in entries: + dominant_inode = host_inode + dominant_count = entries[host_inode] + elif entries: dominant_inode, dominant_count = max(entries.items(), key=lambda kv: kv[1]) + else: + dominant_inode, dominant_count = None, 0 activity = round((dominant_count / total_scanned), 3) if total_scanned > 0 else 0 # Top isolated "worlds" for this namespace (one per inode), richest first. worlds = [ { "inode": inode, "count": count, - "sample": namespace_samples[ns_name].get(inode, []), + "sample": _rank_isolation_samples(namespace_samples[ns_name].get(inode, [])), } for inode, count in sorted(entries.items(), key=lambda kv: kv[1], reverse=True)[:6] ] @@ -971,4 +1016,22 @@ def get_isolation_context(): item["memory_mb_sum"] = round(item["memory_mb_sum"], 1) item.update(stats) - return {"timestamp": datetime.now().isoformat(), "processes_scanned": total_scanned, "namespaces": namespaces, "top_cgroups": top_cgroups} + return { + "timestamp": datetime.now().isoformat(), + "processes_scanned": total_scanned, + "namespaces": namespaces, + "top_cgroups": top_cgroups, + "source": "self", + } + + +def get_isolation_context(): + """Namespace worlds for the ring and the NAMESPACE card. + + Prefers the root collector snapshot (the whole machine). Falls back to + what this process can read itself — usually only its own workers. + """ + snap = _read_isolation_snapshot() + if snap and snap.get("namespaces"): + return snap + return _scan_isolation_context() diff --git a/kernel_ai/services/telemetry_orchestration.py b/kernel_ai/services/telemetry_orchestration.py index 69fc372..5396a82 100644 --- a/kernel_ai/services/telemetry_orchestration.py +++ b/kernel_ai/services/telemetry_orchestration.py @@ -92,7 +92,9 @@ def _ml_anomalies_to_mutations(rows): """Map stored ML anomalies onto the Kernel DNA mutation contract. Keeps only the most recent anomaly per feature (rows arrive newest-first) - and caps the total so the helix is never flooded. + and caps the total so the helix is never flooded. HTTP attempts use a + distinct feature per class (http_attempt:scanner, …) so they do not + collapse into one helix point. """ mutations = [] seen = set() @@ -102,17 +104,31 @@ def _ml_anomalies_to_mutations(rows): continue seen.add(feature) attack = row.get("attack") - if not attack and isinstance(row.get("meta"), dict): - attack = row["meta"].get("attack") + meta = row.get("meta") if isinstance(row.get("meta"), dict) else {} + if not attack: + attack = meta.get("attack") + ml_source = row.get("source") or "ml" + why = meta.get("why") + src_ip = str(meta.get("src_ip") or "") + ip_tail = ".".join(src_ip.split(".")[-2:]) if src_ip else "" + message = row.get("message", "") + if why and why not in message: + message = f"{message} · {why}" if message else str(why) + if ip_tail and "ip …" not in message: + message = f"{message} · ip …{ip_tail}" mut = { "type": feature or row.get("type") or "ml_anomaly", "severity": row.get("severity", "medium"), - "message": row.get("message", ""), - "description": row.get("message", ""), + "message": message, + "description": message, "position": row.get("position", 0.5), "source": "ml", + "ml_source": ml_source, "subsystem": row.get("subsystem"), "score": row.get("score"), + "cls": meta.get("cls"), + "why": why, + "src_ip": src_ip, } if attack: mut["attack"] = attack From 64f416c17269704062efea1d953993cf8644559f Mon Sep 17 00:00:00 2001 From: Aleksei Fedorov Date: Tue, 18 Aug 2026 09:18:41 +0000 Subject: [PATCH 2/2] isolation ui --- static/js/active-connections.js | 11 +- static/js/flow-card.js | 352 ++++++++++++++++++++++++++++++++ static/js/isolation-ui.js | 34 +-- static/js/kernel-dna.js | 12 +- static/js/main.js | 2 +- static/js/namespace-card.js | 273 +++++++++++++++++++++++++ static/js/network-stack.js | 13 ++ static/js/sockets-card.js | 340 ++++++++++++++++++++++++++++++ 8 files changed, 1008 insertions(+), 29 deletions(-) create mode 100644 static/js/flow-card.js create mode 100644 static/js/namespace-card.js create mode 100644 static/js/sockets-card.js diff --git a/static/js/active-connections.js b/static/js/active-connections.js index 1a69944..d2b32b6 100755 --- a/static/js/active-connections.js +++ b/static/js/active-connections.js @@ -118,11 +118,20 @@ class ActiveConnectionsManager { this.attachSocketTooltipHandlers(row, connection); - // Flow follow: open Network stack focused on this remote in FIB/neigh. + // One flow, one card. PATH on the card is the door into the stack. row.on('click', (event) => { if (event && typeof event.stopPropagation === 'function') event.stopPropagation(); const remote = String(connection.remote || '').trim(); if (!remote) return; + const anchor = { + x: 580, + y: startY + 13 + i * 30, + clearOf: 596 + }; + if (window.FlowCard && typeof window.FlowCard.open === 'function') { + window.FlowCard.open(connection, anchor); + return; + } const local = String(connection.local || '').trim(); const type = String(connection.type || 'TCP').toUpperCase(); const params = new URLSearchParams(); diff --git a/static/js/flow-card.js b/static/js/flow-card.js new file mode 100644 index 0000000..4e803b9 --- /dev/null +++ b/static/js/flow-card.js @@ -0,0 +1,352 @@ +// The card a row in the main-page connection list opens. +// +// The list is the index: who is talking to whom. The card is one 4-tuple — +// protocol, state, local end, peer. Traceroute hops sit as lines, not a +// second picture. PATH is the door into the network stack for this flow. +// +// The map chrome stays. This card uses the kcard frame, same as SOCKETS. +const FlowCard = (() => { + const W = 520; + const PAD = 14; + const CUT = 15; + const HEADER = 25; + const LINE = 14; + const ROW_STEP = 16; + const FOOTER = 34; + const MAX_HOPS = 6; + + let openKey = null; + let topKeeper = null; + let requestSeq = 0; + let lastAnchor = null; + let lastFlow = null; + let lastTrace = null; + let layout = null; + + function clip(text, max) { + const value = String(text || ""); + return value.length > max ? `${value.slice(0, max - 1)}…` : value; + } + + function parseEndpoint(endpoint) { + if (!endpoint) return { host: "—", port: "—" }; + const value = String(endpoint); + const separator = value.lastIndexOf(":"); + if (separator === -1) return { host: value, port: "—" }; + return { + host: value.substring(0, separator) || "—", + port: value.substring(separator + 1) || "—" + }; + } + + function stateName(stateCode, protocol) { + if (window.connectionsManager + && typeof window.connectionsManager.getSocketStateName === "function") { + return window.connectionsManager.getSocketStateName(stateCode, protocol); + } + return String(stateCode || "UNKNOWN").toUpperCase(); + } + + function flowKey(connection) { + return [ + String(connection.local || ""), + String(connection.remote || ""), + String(connection.type || "TCP").toUpperCase() + ].join("|"); + } + + function close() { + openKey = null; + lastAnchor = null; + lastFlow = null; + lastTrace = null; + layout = null; + requestSeq += 1; + svg.selectAll(".flow-card-scrim, .flow-card-layer").remove(); + if (topKeeper) topKeeper.stop(); + d3.select("body").on("keydown.flowcard", null); + window.dispatchEvent(new CustomEvent("kcard-closed")); + } + + function door(body, label, x, y, width, onOpen) { + label.style("fill", "#e2a33e"); + const rule = body.append("line") + .attr("x1", x).attr("y1", y + 2.5) + .attr("x2", x + width).attr("y2", y + 2.5) + .attr("stroke", "#e2a33e") + .attr("stroke-width", 1) + .attr("opacity", 0.35); + body.append("rect") + .attr("x", x - 3).attr("y", y - 9) + .attr("width", width + 8).attr("height", 13) + .attr("fill", "transparent") + .style("cursor", "pointer") + .on("mouseenter", () => { + rule.attr("opacity", 1); + }) + .on("mouseleave", () => { + rule.attr("opacity", 0.35); + }) + .on("click", (event) => { + event.stopPropagation(); + onOpen(); + }); + } + + function followPath() { + const flow = lastFlow; + if (!flow) return; + const remote = String(flow.remote || "").trim(); + if (!remote) return; + const local = String(flow.local || "").trim(); + const proto = String(flow.type || "TCP").toUpperCase(); + const parsed = parseEndpoint(remote); + const spec = { + remote, + local, + proto, + ip: parsed.host, + port: Number(parsed.port) || 443 + }; + close(); + window.__flowFollow = spec; + const menu = window.kernelContextMenu; + if (menu && typeof menu.activateNetworkView === "function") { + menu.activateNetworkView(); + const viz = menu.networkVisualization; + if (viz) { + viz.flowFollow = spec; + viz._flowFollowApplied = false; + } + return; + } + const params = new URLSearchParams(); + params.set("remote", remote); + if (local) params.set("local", local); + params.set("proto", proto); + window.location.assign(`/linux-network-subsystem?${params.toString()}`); + } + + function open(connection, anchor) { + if (!connection || !connection.remote) return; + const key = flowKey(connection); + if (openKey === key) { + close(); + return; + } + if (typeof window.closeOpenKernelCards === "function") { + window.closeOpenKernelCards(); + } else { + close(); + } + openKey = key; + lastAnchor = anchor; + lastFlow = connection; + lastTrace = null; + const seq = ++requestSeq; + draw(connection, null, anchor, false); + const remoteIp = parseEndpoint(connection.remote).host; + const fetchTrace = window.connectionsManager + && typeof window.connectionsManager.fetchTraceroute === "function" + ? window.connectionsManager.fetchTraceroute(remoteIp) + : fetch(`/api/traceroute?ip=${encodeURIComponent(remoteIp)}`) + .then((r) => r.json()) + .catch(() => ({ note: "traceroute unavailable", hops: [] })); + Promise.resolve(fetchTrace).then((trace) => { + if (seq !== requestSeq || openKey !== key) return; + lastTrace = trace; + draw(connection, trace, lastAnchor, true); + }); + } + + function cardHeight(trace) { + let h = HEADER + 12 + 10; + h += LINE; + h += 16 + LINE + LINE; + h += 16 + LINE + LINE; + h += 16 + LINE; + const hops = Array.isArray(trace && trace.hops) ? trace.hops.slice(0, MAX_HOPS) : []; + if (trace && trace.note && !hops.length) h += LINE; + else if (!trace) h += LINE; + else h += Math.max(1, hops.length) * ROW_STEP; + h += FOOTER; + return h; + } + + function draw(connection, trace, anchor, live) { + const svgNode = svg.node(); + const viewW = (svgNode && svgNode.clientWidth) || window.innerWidth; + const viewH = (svgNode && svgNode.clientHeight) || window.innerHeight; + const cw = (live && layout) ? layout.cw : Math.min(W, viewW - 24); + const compact = cw < 420; + const h = cardHeight(trace); + + let x; + let y; + if (live && layout) { + x = layout.x; + y = Math.max(12, Math.min(viewH - h - 12, layout.y)); + } else { + const from = anchor && Number.isFinite(anchor.x) ? anchor.x : 300; + x = Number.isFinite(anchor && anchor.clearOf) ? anchor.clearOf : from + 40; + if (x + cw + 16 > viewW) x = Math.max(12, from - cw - 40); + if (x < 12) x = Math.max(12, viewW - cw - 16); + y = (anchor && Number.isFinite(anchor.y) ? anchor.y : 120) - 20; + y = Math.max(12, Math.min(viewH - h - 12, y)); + } + layout = { x, y, cw, h }; + + let layer; + let panel; + if (live) { + layer = svg.select(".flow-card-layer"); + panel = layer.select(".flow-card-panel"); + if (layer.empty() || panel.empty()) return; + panel.attr("transform", `translate(${x}, ${y})`); + panel.select(".kcard-frame").attr("d", dossierCardPath(0, 0, cw, h, CUT)); + if (anchor && Number.isFinite(anchor.x) && Number.isFinite(anchor.y)) { + const connY = Math.max(y + 12, Math.min(y + h - 12, anchor.y)); + layer.select(".kcard-conn").attr("x2", x).attr("y2", connY); + } + panel.select(".flow-card-body").remove(); + } else { + ensureDossierDefs(); + svg.append("rect") + .attr("class", "flow-card-scrim") + .attr("x", 0).attr("y", 0).attr("width", viewW).attr("height", viewH) + .attr("fill", ensureFocusVeilGradient()) + .style("opacity", 0) + .style("cursor", "pointer") + .on("click", () => close()) + .transition().duration(200).style("opacity", 1); + + layer = svg.append("g").attr("class", "flow-card-layer"); + if (!topKeeper) { + topKeeper = createOverlayTopKeeper("flow-card-scrim", ["flow-card-layer"], () => openKey !== null); + } + topKeeper.start(); + + if (anchor && Number.isFinite(anchor.x) && Number.isFinite(anchor.y)) { + const connY = Math.max(y + 12, Math.min(y + h - 12, anchor.y)); + layer.append("circle") + .attr("class", "kcard-anchor") + .attr("cx", anchor.x).attr("cy", anchor.y).attr("r", 3); + layer.append("line") + .attr("class", "kcard-conn") + .attr("x1", anchor.x).attr("y1", anchor.y) + .attr("x2", anchor.x).attr("y2", anchor.y) + .transition().duration(220).ease(d3.easeCubicOut) + .attr("x2", x).attr("y2", connY); + } + + panel = layer.append("g") + .attr("class", "flow-card-panel") + .attr("transform", `translate(${x}, ${y})`) + .on("click", (event) => event.stopPropagation()); + + panel.append("path") + .attr("class", "kcard-frame") + .attr("d", dossierCardPath(0, 0, cw, h, CUT)) + .attr("filter", "url(#dossier-drop)") + .attr("transform", `translate(0, ${h / 2}) scale(1, 0.02)`) + .transition().delay(120).duration(200).ease(d3.easeCubicOut) + .attr("transform", "translate(0,0) scale(1,1)"); + } + + const body = panel.append("g").attr("class", "flow-card-body"); + if (!live) { + body.style("opacity", 0); + body.transition().delay(250).duration(180).style("opacity", 1); + } + + paintBody(body, connection, trace, cw, compact, h); + d3.select("body").on("keydown.flowcard", (event) => { + if (event.key === "Escape") close(); + }); + } + + function paintBody(body, connection, trace, cw, compact, h) { + const proto = String(connection.type || "TCP").toUpperCase(); + const state = stateName(connection.state, proto); + const local = parseEndpoint(connection.local); + const remote = parseEndpoint(connection.remote); + const hops = Array.isArray(trace && trace.hops) ? trace.hops.slice(0, MAX_HOPS) : []; + + const text = (cls, tx, ty, value, anchorEnd) => body.append("text") + .attr("class", cls) + .attr("x", tx).attr("y", ty) + .attr("text-anchor", anchorEnd ? "end" : "start") + .text(value); + + body.append("path") + .attr("class", "kcard-strip") + .attr("d", `M0,0 H${cw - CUT} L${cw},${CUT} V${HEADER} H0 Z`); + body.append("circle") + .attr("class", "kcard-glyph-ring") + .attr("cx", PAD).attr("cy", HEADER / 2).attr("r", 4.2); + body.append("circle") + .attr("class", "kcard-glyph-dot") + .attr("cx", PAD).attr("cy", HEADER / 2).attr("r", 1.6); + text("kcard-title", PAD + 12, HEADER / 2 + 3.5, "FLOW"); + text("kcard-meta", cw - 13, HEADER / 2 + 3.5, + `${proto} · ${clip(state, 10)}`, true) + .style("fill", "rgba(244, 244, 236, 0.5)"); + body.append("line") + .attr("class", "kcard-divider") + .attr("x1", 0).attr("y1", HEADER).attr("x2", cw).attr("y2", HEADER); + + let cy = HEADER + 12 + 10; + text("kcard-line", PAD, cy, compact + ? "one 4-tuple is the picture" + : "one 4-tuple is the picture — not the whole stack"); + cy += LINE; + + cy += 16; + text("kcard-section", PAD, cy, "LOCAL"); + cy += LINE; + text("kcard-waiter", PAD, cy, clip(`${local.host}:${local.port}`, compact ? 36 : 52)); + cy += LINE; + + cy += 16; + text("kcard-section", PAD, cy, "PEER"); + cy += LINE; + text("kcard-waiter", PAD, cy, clip(`${remote.host}:${remote.port}`, compact ? 36 : 52)); + cy += LINE; + + cy += 16; + text("kcard-section", PAD, cy, "THE WIRE"); + cy += LINE; + if (!trace) { + text("kcard-faint", PAD, cy, "TRACEROUTE · LOADING"); + cy += LINE; + } else if (hops.length) { + hops.forEach((hop) => { + const rtt = hop.rtt_ms === null || hop.rtt_ms === undefined + ? "*" + : `${Number(hop.rtt_ms).toFixed(1)} ms`; + text("kcard-waiter-dim", PAD, cy + 4, + clip(`${hop.hop}. ${hop.target} ${rtt}`, compact ? 40 : 56)); + cy += ROW_STEP; + }); + } else { + text("kcard-faint", PAD, cy, clip( + (trace && trace.note) || "NO HOPS FROM HERE", + compact ? 40 : 56 + )); + cy += LINE; + } + + body.append("line") + .attr("class", "kcard-divider") + .attr("x1", 0).attr("y1", h - FOOTER + 8).attr("x2", cw).attr("y2", h - FOOTER + 8); + text("kcard-foot", PAD, h - 10, "ESC OR CLICK OUTSIDE TO CLOSE"); + const pathLabel = text("kcard-signature", cw - PAD, h - 10, "PATH", true) + .attr("letter-spacing", 1.2); + const pathBox = pathLabel.node().getBBox(); + door(body, pathLabel, pathBox.x, h - 10, pathBox.width, followPath); + } + + return { open, close, isOpen: () => openKey !== null }; +})(); + +window.FlowCard = FlowCard; diff --git a/static/js/isolation-ui.js b/static/js/isolation-ui.js index 95da9bb..652ecf7 100644 --- a/static/js/isolation-ui.js +++ b/static/js/isolation-ui.js @@ -163,10 +163,7 @@ function drawNamespaceShell(centerX, centerY, namespaces) { const activity = Math.max(0, Math.min(1, Number(ns.activity || 0))); const meta = NS_META[ns.id] || {}; const nsName = ns.label || meta.name || String(ns.id || 'NS').toUpperCase(); - // A cell with more than one distinct inode contains real isolation - // (containers / sandboxes) — the most security-relevant signal. const isolated = !!ns.isolated || Number(ns.unique_count || 0) > 1; - const ACCENT = '88, 182, 216'; const kind = NS_KIND[ns.id] || 'nsproxy'; // Soft focus halo behind the segment (hidden until hover). @@ -182,29 +179,13 @@ function drawNamespaceShell(centerX, centerY, namespaces) { const segment = shellGroup.append('path') .attr('d', dPath) .attr('transform', `translate(${centerX}, ${centerY})`) - .attr('fill', isolated - ? `rgba(${ACCENT}, ${0.08 + activity * 0.2})` - : `rgba(${INK}, ${0.06 + activity * 0.22})`) - .attr('stroke', isolated - ? `rgba(${ACCENT}, ${0.7 + activity * 0.3})` - : `rgba(${INK}, ${0.42 + activity * 0.4})`) - .attr('stroke-width', isolated ? 1.8 + activity * 1.4 : 1.1 + activity * 1.6) + .attr('fill', `rgba(${INK}, 0.07)`) + .attr('stroke', `rgba(${INK}, 0.44)`) + .attr('stroke-width', 1.1) .style('cursor', 'pointer'); const mid = (startAngle + endAngle) / 2; - // Pulsing marker flags cells that actually contain isolation. - if (isolated) { - const markR = ringOuter + 6; - shellGroup.append('circle') - .attr('class', 'ns-isolated-marker') - .attr('cx', centerX + Math.cos(mid - Math.PI / 2) * markR) - .attr('cy', centerY + Math.sin(mid - Math.PI / 2) * markR) - .attr('r', 2.6) - .attr('fill', `rgb(${ACCENT})`) - .style('pointer-events', 'none'); - } - const idx = cells.length; cells.push({ segment, halo }); @@ -230,7 +211,7 @@ function drawNamespaceShell(centerX, centerY, namespaces) {
INODE${ns.dominant_inode || 'n/a'}
VIA${kind}
-
ACTIVITY ${Math.round(activity * 100)}%CLICK TO UNFOLD ▸
+
ACTIVITY ${Math.round(activity * 100)}%CLICK FOR THE CARD ▸
`); @@ -246,6 +227,13 @@ function drawNamespaceShell(centerX, centerY, namespaces) { .on('click', (event) => { event.stopPropagation(); d3.selectAll('.ns-tooltip').remove(); + const ax = centerX + Math.cos(mid - Math.PI / 2) * 202; + const ay = centerY + Math.sin(mid - Math.PI / 2) * 202; + if (window.NamespaceCard && typeof window.NamespaceCard.open === 'function') { + restoreFocus(); + window.NamespaceCard.open(ns, { x: ax, y: ay }); + return; + } if (expandedNsId === ns.id) { collapseNamespaceTree(); } else { diff --git a/static/js/kernel-dna.js b/static/js/kernel-dna.js index 613d1e0..e499e56 100755 --- a/static/js/kernel-dna.js +++ b/static/js/kernel-dna.js @@ -435,6 +435,9 @@ class KernelDNAVisualization { // ml -> statistical baseline detector (cyan wireframe) // Severity drives the assessment marker colour (high = red, else yellow). const isML = mutationData.source === 'ml'; + const typeStr = String(mutationData.type || 'anomaly'); + const isHttpAttempt = typeStr.startsWith('http_attempt'); + const isHttpSuccess = typeStr.startsWith('http_success'); const isHigh = String(mutationData.severity || '').toLowerCase() === 'high'; const attack = mutationData.attack || null; const attackColorCss = (attack && attack.color) ? String(attack.color) : null; @@ -474,11 +477,12 @@ class KernelDNAVisualization { yellowMarker.position.y += 0.3; // Position above mutation const mitreTag = attack && attack.mitre ? String(attack.mitre) : ''; - const baseLabel = (mitreTag - ? mitreTag - : String(mutationData.type || 'anomaly').replace(/_/g, ' ') + const httpLabel = typeStr.replace(/^http_(attempt|success):/, '').replace(/_/g, ' '); + const baseLabel = (isHttpAttempt || isHttpSuccess + ? httpLabel + : (mitreTag ? mitreTag : typeStr.replace(/_/g, ' ')) ).toUpperCase().slice(0, 22); - const tag = isML ? (mitreTag ? 'ATT' : 'ML') : 'RULE'; + const tag = isHttpSuccess ? 'HIT' : (isHttpAttempt ? 'HTTP' : (isML ? (mitreTag ? 'ATT' : 'ML') : 'RULE')); const canvas = document.createElement('canvas'); canvas.width = 256; canvas.height = 48; diff --git a/static/js/main.js b/static/js/main.js index 7627ed3..96b1f9d 100755 --- a/static/js/main.js +++ b/static/js/main.js @@ -1695,7 +1695,7 @@ const processModalTopKeeper = createOverlayTopKeeper( function closeOpenKernelCards() { ["MemoryCard", "ThreadsCard", "WaitsCard", "WakeupsCard", "SocketsCard", - "SyscallCard", "IrqCard", "RunqueueCard"].forEach((name) => { + "FlowCard", "NamespaceCard", "SyscallCard", "IrqCard", "RunqueueCard"].forEach((name) => { const card = window[name]; if (card && typeof card.close === "function") card.close(); }); diff --git a/static/js/namespace-card.js b/static/js/namespace-card.js new file mode 100644 index 0000000..6f0de3c --- /dev/null +++ b/static/js/namespace-card.js @@ -0,0 +1,273 @@ +// The card a namespace slice on the host ring opens. +// +// The ring is the architecture: seven kinds of world around the processes. +// The card is one kind — what it cuts off, how many worlds live on this +// host, who left the host world. A process name is a door into its dossier. +const NamespaceCard = (() => { + const W = 520; + const PAD = 14; + const CUT = 15; + const HEADER = 25; + const LINE = 14; + const ROW_STEP = 16; + const FOOTER = 34; + const MAX_WORLDS = 6; + + const MEANING = { + mnt: "the filesystem tree this process is allowed to see", + pid: "its own process tree — PID 1 is not the host's", + net: "interfaces, ports and routes of one network world", + ipc: "queues and shm that stay inside this world", + uts: "hostname and domain this world answers to", + user: "which UIDs and capabilities it believes it has", + cgroup: "which cgroup tree is the root it can see", + time: "boot and monotonic clocks of this world" + }; + + let openKey = null; + let topKeeper = null; + let lastAnchor = null; + let lastNs = null; + let layout = null; + + function clip(text, max) { + const value = String(text || ""); + return value.length > max ? `${value.slice(0, max - 1)}…` : value; + } + + function close() { + openKey = null; + lastAnchor = null; + lastNs = null; + layout = null; + svg.selectAll(".namespace-card-scrim, .namespace-card-layer").remove(); + if (topKeeper) topKeeper.stop(); + d3.select("body").on("keydown.nscard", null); + window.dispatchEvent(new CustomEvent("kcard-closed")); + } + + function door(body, label, x, y, width, onOpen) { + label.style("fill", "#e2a33e"); + const rule = body.append("line") + .attr("x1", x).attr("y1", y + 2.5) + .attr("x2", x + width).attr("y2", y + 2.5) + .attr("stroke", "#e2a33e") + .attr("stroke-width", 1) + .attr("opacity", 0.35); + body.append("rect") + .attr("x", x - 3).attr("y", y - 9) + .attr("width", width + 8).attr("height", 13) + .attr("fill", "transparent") + .style("cursor", "pointer") + .on("mouseenter", () => rule.attr("opacity", 1)) + .on("mouseleave", () => rule.attr("opacity", 0.35)) + .on("click", (event) => { + event.stopPropagation(); + onOpen(); + }); + } + + function followProcess(name) { + if (!name) return; + close(); + if (typeof window.openProcessDossier === "function") { + window.openProcessDossier({ name }); + } + } + + function open(ns, anchor) { + if (!ns || !ns.id) return; + const key = String(ns.id); + if (openKey === key) { + close(); + return; + } + if (typeof window.collapseNamespaceTree === "function") { + window.collapseNamespaceTree(false); + } + if (typeof window.closeOpenKernelCards === "function") { + window.closeOpenKernelCards(); + } else { + close(); + } + openKey = key; + lastAnchor = anchor; + lastNs = ns; + draw(ns, anchor); + } + + function worldsOf(ns) { + return (Array.isArray(ns.worlds) ? ns.worlds : []).slice(0, MAX_WORLDS); + } + + function cardHeight(ns) { + const worlds = worldsOf(ns); + let h = HEADER + 12 + 10; + h += LINE + LINE; + h += 16 + LINE; + if (!worlds.length) h += LINE; + else h += worlds.length * ROW_STEP; + const hidden = Math.max(0, Number(ns.unique_count || 0) - worlds.length); + if (hidden) h += LINE; + h += FOOTER; + return h; + } + + function draw(ns, anchor) { + const svgNode = svg.node(); + const viewW = (svgNode && svgNode.clientWidth) || window.innerWidth; + const viewH = (svgNode && svgNode.clientHeight) || window.innerHeight; + const cw = Math.min(W, viewW - 24); + const compact = cw < 420; + const h = cardHeight(ns); + + const from = anchor && Number.isFinite(anchor.x) ? anchor.x : viewW * 0.5; + let x = Number.isFinite(anchor && anchor.clearOf) ? anchor.clearOf : from + 28; + if (x + cw + 16 > viewW) x = Math.max(12, from - cw - 28); + if (x < 12) x = Math.max(12, viewW - cw - 16); + let y = (anchor && Number.isFinite(anchor.y) ? anchor.y : 120) - 24; + y = Math.max(12, Math.min(viewH - h - 12, y)); + layout = { x, y, cw, h }; + + svg.selectAll(".namespace-card-scrim, .namespace-card-layer").remove(); + ensureDossierDefs(); + svg.append("rect") + .attr("class", "namespace-card-scrim") + .attr("x", 0).attr("y", 0).attr("width", viewW).attr("height", viewH) + .attr("fill", ensureFocusVeilGradient()) + .style("opacity", 0) + .style("cursor", "pointer") + .on("click", () => close()) + .transition().duration(200).style("opacity", 1); + + const layer = svg.append("g").attr("class", "namespace-card-layer"); + if (!topKeeper) { + topKeeper = createOverlayTopKeeper( + "namespace-card-scrim", + ["namespace-card-layer"], + () => openKey !== null + ); + } + topKeeper.start(); + + if (anchor && Number.isFinite(anchor.x) && Number.isFinite(anchor.y)) { + const connY = Math.max(y + 12, Math.min(y + h - 12, anchor.y)); + layer.append("circle") + .attr("class", "kcard-anchor") + .attr("cx", anchor.x).attr("cy", anchor.y).attr("r", 3); + layer.append("line") + .attr("class", "kcard-conn") + .attr("x1", anchor.x).attr("y1", anchor.y) + .attr("x2", anchor.x).attr("y2", anchor.y) + .transition().duration(220).ease(d3.easeCubicOut) + .attr("x2", x).attr("y2", connY); + } + + const panel = layer.append("g") + .attr("class", "namespace-card-panel") + .attr("transform", `translate(${x}, ${y})`) + .on("click", (event) => event.stopPropagation()); + + panel.append("path") + .attr("class", "kcard-frame") + .attr("d", dossierCardPath(0, 0, cw, h, CUT)) + .attr("filter", "url(#dossier-drop)") + .attr("transform", `translate(0, ${h / 2}) scale(1, 0.02)`) + .transition().delay(120).duration(200).ease(d3.easeCubicOut) + .attr("transform", "translate(0,0) scale(1,1)"); + + const body = panel.append("g").attr("class", "namespace-card-body") + .style("opacity", 0); + body.transition().delay(250).duration(180).style("opacity", 1); + paintBody(body, ns, cw, compact, h); + d3.select("body").on("keydown.nscard", (event) => { + if (event.key === "Escape") close(); + }); + } + + function paintBody(body, ns, cw, compact, h) { + const isolated = !!ns.isolated || Number(ns.unique_count || 0) > 1; + const name = String(ns.label || ns.id || "NS").toUpperCase(); + const meaning = MEANING[ns.id] || "a separate world for one kernel resource"; + const worlds = worldsOf(ns); + const hostInode = ns.dominant_inode; + const hidden = Math.max(0, Number(ns.unique_count || 0) - worlds.length); + + const text = (cls, tx, ty, value, anchorEnd) => body.append("text") + .attr("class", cls) + .attr("x", tx).attr("y", ty) + .attr("text-anchor", anchorEnd ? "end" : "start") + .text(value); + + body.append("path") + .attr("class", "kcard-strip") + .attr("d", `M0,0 H${cw - CUT} L${cw},${CUT} V${HEADER} H0 Z`); + body.append("circle") + .attr("class", "kcard-glyph-ring") + .attr("cx", PAD).attr("cy", HEADER / 2).attr("r", 4.2); + body.append("circle") + .attr("class", "kcard-glyph-dot") + .attr("cx", PAD).attr("cy", HEADER / 2).attr("r", 1.6); + text("kcard-title", PAD + 12, HEADER / 2 + 3.5, `NAMESPACE · ${name}`); + text("kcard-meta", cw - 13, HEADER / 2 + 3.5, + isolated ? "ISOLATED" : "ONE WORLD", true) + .style("fill", isolated ? "#e2a33e" : "rgba(244, 244, 236, 0.5)"); + body.append("line") + .attr("class", "kcard-divider") + .attr("x1", 0).attr("y1", HEADER).attr("x2", cw).attr("y2", HEADER); + + let cy = HEADER + 12 + 10; + text("kcard-line", PAD, cy, clip(meaning, compact ? 42 : 62)); + cy += LINE; + text("kcard-faint", PAD, cy, isolated + ? "more than one world — someone left the host" + : "every process shares this world with the host"); + cy += LINE; + + cy += 16; + text("kcard-section", PAD, cy, + `WORLDS ON THIS HOST · ${ns.unique_count || worlds.length}`); + cy += LINE; + + if (!worlds.length) { + text("kcard-faint", PAD, cy, "NO WORLD TABLE FOR THIS NAMESPACE"); + cy += LINE; + } + + worlds.forEach((world) => { + const ty = cy + 4; + const names = (Array.isArray(world.sample) ? world.sample : []).filter(Boolean).slice(0, 3); + const isHost = hostInode != null && String(world.inode) === String(hostInode); + text(isHost ? "kcard-faint" : "kcard-waiter", PAD, ty, isHost ? "HOST" : "LEFT"); + const countText = text("kcard-waiter-dim", PAD + 52, ty, `${world.count}p · `); + let x = PAD + 52 + countText.node().getBBox().width; + if (!names.length) { + text("kcard-waiter-dim", x, ty, "—"); + } + names.forEach((name, idx) => { + if (idx) { + const sep = text("kcard-faint", x, ty, " · "); + x += sep.node().getBBox().width; + } + const who = text("kcard-waiter-dim", x, ty, clip(name, compact ? 10 : 16)); + const box = who.node().getBBox(); + door(body, who, box.x, ty, box.width, () => followProcess(name)); + x += box.width; + }); + cy += ROW_STEP; + }); + if (hidden) { + text("kcard-faint", PAD, cy, `AND ${hidden} SMALLER ${hidden === 1 ? "WORLD" : "WORLDS"}`); + } + + body.append("line") + .attr("class", "kcard-divider") + .attr("x1", 0).attr("y1", h - FOOTER + 8).attr("x2", cw).attr("y2", h - FOOTER + 8); + text("kcard-foot", PAD, h - 10, "ESC OR CLICK OUTSIDE TO CLOSE"); + text("kcard-foot", cw - PAD, h - 10, `/PROC/*/NS/${String(ns.id || "").toUpperCase()}`, true); + } + + return { open, close, isOpen: () => openKey !== null }; +})(); + +window.NamespaceCard = NamespaceCard; diff --git a/static/js/network-stack.js b/static/js/network-stack.js index ad4be97..1aaf10a 100644 --- a/static/js/network-stack.js +++ b/static/js/network-stack.js @@ -3629,6 +3629,19 @@ class NetworkStackVisualization { parseFlowFollowFromUrl() { try { + const handed = window.__flowFollow; + if (handed && handed.remote) { + const { ip, port } = this.parseEndpoint(handed.remote); + if (ip) { + return { + remote: handed.remote, + local: handed.local || '', + proto: String(handed.proto || 'TCP').toUpperCase(), + ip: handed.ip || ip, + port: handed.port != null ? handed.port : (port != null ? port : 443) + }; + } + } const params = new URLSearchParams(window.location.search || ''); const remote = String(params.get('remote') || '').trim(); if (!remote) return null; diff --git a/static/js/sockets-card.js b/static/js/sockets-card.js new file mode 100644 index 0000000..8d295de --- /dev/null +++ b/static/js/sockets-card.js @@ -0,0 +1,340 @@ +// The card the NET CONNS tile of the process dossier opens. +// +// The tile counts inet connections. The card is the sockets themselves: state, +// the local end, the peer. If the other end is a process we can name, that +// name is a door into its dossier — the same gesture as parked_in. +// +// /proc//fd plus /proc/net name a socket when ptrace will not. A missing +// table is said, not filled in. +// +// While the card is open it re-reads the same payload every couple of seconds +// and repaints the body. The frame stays put. +const SocketsCard = (() => { + const W = 600; + const PAD = 14; + const CUT = 15; + const HEADER = 25; + const LINE = 14; + const ROW_STEP = 16; + const FOOTER = 34; + const MAX_ROWS = 12; + const POLL_MS = 2000; + const COL_STATE = PAD; + const COL_LOCAL = 78; + const COL_PEER = 280; + + let openPid = null; + let topKeeper = null; + let requestSeq = 0; + let pollTimer = null; + let lastAnchor = null; + let layout = null; + + function clip(text, max) { + const value = String(text || ""); + return value.length > max ? `${value.slice(0, max - 1)}…` : value; + } + + function stateLabel(value) { + const raw = String(value || "").toUpperCase(); + if (raw === "ESTABLISHED") return "ESTAB"; + if (raw === "TIME_WAIT") return "TIMEW"; + if (raw === "CLOSE_WAIT") return "CWAIT"; + if (raw === "FIN_WAIT1" || raw === "FIN_WAIT2") return "FINW"; + return raw ? raw.slice(0, 6) : "—"; + } + + function stopPoll() { + if (pollTimer) { + clearInterval(pollTimer); + pollTimer = null; + } + } + + function close() { + stopPoll(); + openPid = null; + lastAnchor = null; + layout = null; + requestSeq += 1; + svg.selectAll(".sockets-card-scrim, .sockets-card-layer").remove(); + if (topKeeper) topKeeper.stop(); + d3.select("body").on("keydown.socketscard", null); + window.dispatchEvent(new CustomEvent("kcard-closed")); + } + + function startPoll(pid) { + stopPoll(); + pollTimer = setInterval(() => { + if (openPid !== pid) { + stopPoll(); + return; + } + if (document.hidden) return; + const seq = requestSeq; + fetch(`/api/process/${pid}/fds`, { cache: "no-store" }) + .then((r) => r.json()) + .then((data) => { + if (seq !== requestSeq || openPid !== pid) return; + if (!data || data.error) { + close(); + return; + } + draw(data, lastAnchor, true); + }) + .catch(() => {}); + }, POLL_MS); + } + + function open(pid, anchor) { + const key = Number(pid); + if (!Number.isFinite(key)) return; + if (openPid === key) { + close(); + return; + } + close(); + openPid = key; + lastAnchor = anchor; + const seq = ++requestSeq; + fetch(`/api/process/${key}/fds`, { cache: "no-store" }) + .then((r) => r.json()) + .then((data) => { + if (seq !== requestSeq) return; + if (!data || data.error) { + openPid = null; + return; + } + draw(data, anchor, false); + startPoll(key); + }) + .catch((err) => { + if (seq !== requestSeq) return; + openPid = null; + if (window.frontendLogger) { + window.frontendLogger.error("sockets card failed to draw", { + source: "sockets-card", stack: String((err && err.stack) || err) + }); + } + }); + } + + function socketRows(data) { + return (data.connections || []).slice(0, MAX_ROWS); + } + + function cardHeight(data, compact) { + const rows = socketRows(data); + let h = HEADER + 12 + 10; + h += LINE; + h += 16 + LINE; + if (!rows.length) h += LINE + LINE; + else h += LINE + rows.length * ROW_STEP; + h += FOOTER; + return h; + } + + function followProcess(pid, name) { + close(); + if (typeof window.openProcessDossier === "function") { + window.openProcessDossier({ pid, name }); + } + } + + function door(body, label, x, y, width, onOpen) { + const rule = body.append("line") + .attr("x1", x).attr("y1", y + 2.5) + .attr("x2", x + width).attr("y2", y + 2.5) + .attr("stroke", "#e2a33e") + .attr("stroke-width", 1) + .attr("opacity", 0.35); + body.append("rect") + .attr("x", x - 3).attr("y", y - 9) + .attr("width", width + 8).attr("height", 13) + .attr("fill", "transparent") + .style("cursor", "pointer") + .on("mouseenter", () => { + label.attr("fill", "#e2a33e"); + rule.attr("opacity", 1); + }) + .on("mouseleave", () => { + label.attr("fill", null); + rule.attr("opacity", 0.35); + }) + .on("click", (event) => { + event.stopPropagation(); + onOpen(); + }); + } + + function draw(data, anchor, live) { + const svgNode = svg.node(); + const viewW = (svgNode && svgNode.clientWidth) || window.innerWidth; + const viewH = (svgNode && svgNode.clientHeight) || window.innerHeight; + const cw = (live && layout) ? layout.cw : Math.min(W, viewW - 24); + const compact = cw < 460; + const h = cardHeight(data, compact); + + let x; + let y; + if (live && layout) { + x = layout.x; + y = Math.max(12, Math.min(viewH - h - 12, layout.y)); + } else { + const from = anchor && Number.isFinite(anchor.x) ? anchor.x : 300; + x = Number.isFinite(anchor && anchor.clearOf) ? anchor.clearOf : from + 40; + if (x + cw + 16 > viewW) x = Math.max(12, from - cw - 40); + if (x < 12) x = Math.max(12, viewW - cw - 16); + y = (anchor && Number.isFinite(anchor.y) ? anchor.y : 120) - 30; + y = Math.max(12, Math.min(viewH - h - 12, y)); + } + layout = { x, y, cw, h }; + + let layer; + let panel; + if (live) { + layer = svg.select(".sockets-card-layer"); + panel = layer.select(".sockets-card-panel"); + if (layer.empty() || panel.empty()) return; + panel.attr("transform", `translate(${x}, ${y})`); + panel.select(".kcard-frame").attr("d", dossierCardPath(0, 0, cw, h, CUT)); + if (anchor && Number.isFinite(anchor.x) && Number.isFinite(anchor.y)) { + const connY = Math.max(y + 12, Math.min(y + h - 12, anchor.y)); + layer.select(".kcard-conn").attr("x2", x).attr("y2", connY); + } + panel.select(".sockets-card-body").remove(); + } else { + ensureDossierDefs(); + svg.append("rect") + .attr("class", "sockets-card-scrim") + .attr("x", 0).attr("y", 0).attr("width", viewW).attr("height", viewH) + .attr("fill", ensureFocusVeilGradient()) + .style("opacity", 0) + .style("cursor", "pointer") + .on("click", () => close()) + .transition().duration(200).style("opacity", 1); + + layer = svg.append("g").attr("class", "sockets-card-layer"); + if (!topKeeper) { + topKeeper = createOverlayTopKeeper("sockets-card-scrim", ["sockets-card-layer"], () => openPid !== null); + } + topKeeper.start(); + + if (anchor && Number.isFinite(anchor.x) && Number.isFinite(anchor.y)) { + const connY = Math.max(y + 12, Math.min(y + h - 12, anchor.y)); + layer.append("circle") + .attr("class", "kcard-anchor") + .attr("cx", anchor.x).attr("cy", anchor.y).attr("r", 3); + layer.append("line") + .attr("class", "kcard-conn") + .attr("x1", anchor.x).attr("y1", anchor.y) + .attr("x2", anchor.x).attr("y2", anchor.y) + .transition().duration(220).ease(d3.easeCubicOut) + .attr("x2", x).attr("y2", connY); + } + + panel = layer.append("g") + .attr("class", "sockets-card-panel") + .attr("transform", `translate(${x}, ${y})`) + .on("click", (event) => event.stopPropagation()); + + panel.append("path") + .attr("class", "kcard-frame") + .attr("d", dossierCardPath(0, 0, cw, h, CUT)) + .attr("filter", "url(#dossier-drop)") + .attr("transform", `translate(0, ${h / 2}) scale(1, 0.02)`) + .transition().delay(120).duration(200).ease(d3.easeCubicOut) + .attr("transform", "translate(0,0) scale(1,1)"); + } + + const body = panel.append("g").attr("class", "sockets-card-body"); + if (!live) { + body.style("opacity", 0); + body.transition().delay(250).duration(180).style("opacity", 1); + } + + paintBody(body, data, cw, compact, h); + d3.select("body").on("keydown.socketscard", (event) => { + if (event.key === "Escape") close(); + }); + } + + function paintBody(body, data, cw, compact, h) { + const rows = socketRows(data); + const text = (cls, tx, ty, value, anchorEnd) => body.append("text") + .attr("class", cls) + .attr("x", tx).attr("y", ty) + .attr("text-anchor", anchorEnd ? "end" : "start") + .text(value); + + body.append("path") + .attr("class", "kcard-strip") + .attr("d", `M0,0 H${cw - CUT} L${cw},${CUT} V${HEADER} H0 Z`); + body.append("circle") + .attr("class", "kcard-glyph-ring") + .attr("cx", PAD).attr("cy", HEADER / 2).attr("r", 4.2); + body.append("circle") + .attr("class", "kcard-glyph-dot") + .attr("cx", PAD).attr("cy", HEADER / 2).attr("r", 1.6); + text("kcard-title", PAD + 12, HEADER / 2 + 3.5, "SOCKETS"); + text("kcard-meta", cw - 13, HEADER / 2 + 3.5, + `PID ${data.pid} · ${rows.length} LIVE`, true) + .style("fill", "rgba(244, 244, 236, 0.5)"); + body.append("line") + .attr("class", "kcard-divider") + .attr("x1", 0).attr("y1", HEADER).attr("x2", cw).attr("y2", HEADER); + + let cy = HEADER + 12 + 10; + text("kcard-line", PAD, cy, compact + ? "state, local end, and the peer" + : "a 5-tuple plus state is the picture"); + cy += LINE; + + cy += 16; + text("kcard-section", PAD, cy, "STATE LOCAL PEER"); + cy += LINE; + + if (!rows.length) { + text("kcard-faint", PAD, cy, "NO SOCKETS ARE VISIBLE FOR THIS PROCESS"); + cy += LINE; + text("kcard-faint", PAD, cy, "THE TILE COUNTED WHAT /PROC WOULD SHOW WHEN IT COULD"); + cy += LINE; + } + + const localChars = compact ? 18 : 24; + const peerChars = compact ? 16 : 22; + rows.forEach((row) => { + const ty = cy + 4; + const family = String(row.family || "").toLowerCase(); + const local = clip(row.local_address || "—", localChars); + const remote = row.remote_address ? clip(row.remote_address, peerChars) : "—"; + const peerName = row.peer_name ? clip(row.peer_name, 12) : ""; + const peerText = peerName || remote; + text("kcard-waiter", COL_STATE, ty, stateLabel(row.status)); + text("kcard-waiter-dim", COL_LOCAL, ty, local); + const peer = text("kcard-faint", compact ? COL_LOCAL + 160 : COL_PEER, ty, + family === "unix" && !row.remote_address + ? `unix${peerName ? ` · ${peerName}` : ""}` + : peerText); + const peerPid = Number(row.peer_pid); + const canFollow = Number.isFinite(peerPid) && peerPid > 0 && peerPid !== Number(data.pid); + if (canFollow) { + const box = peer.node().getBBox(); + door(body, peer, box.x, ty, box.width, () => { + followProcess(peerPid, row.peer_name); + }); + } + cy += ROW_STEP; + }); + + body.append("line") + .attr("class", "kcard-divider") + .attr("x1", 0).attr("y1", h - FOOTER + 8).attr("x2", cw).attr("y2", h - FOOTER + 8); + text("kcard-foot", PAD, h - 10, "ESC OR CLICK OUTSIDE TO CLOSE"); + text("kcard-foot", cw - PAD, h - 10, `/PROC/${data.pid}/NET`, true); + } + + return { open, close, isOpen: () => openPid !== null }; +})(); + +window.SocketsCard = SocketsCard;