ML module init

kernel_ai/api/rest.py

@@ -28,6 +28,7 @@
     ("/proc-timeline-branches", "get_proc_timeline_branches", h.get_proc_timeline_branches, None),
     ("/execution-context", "get_execution_context", h.get_execution_context, None),
     ("/kernel-dna", "kernel_dna", h.kernel_dna, None),
+    ("/ml-anomalies", "ml_anomalies", h.ml_anomalies, None),
     ("/crypto-realtime", "crypto_realtime", h.crypto_realtime, None),
     ("/security-realtime", "security_realtime", h.security_realtime, None),
     ("/processes-realtime", "processes_realtime", h.processes_realtime, None),

kernel_ai/http/api.py

@@ -6,6 +6,7 @@
     io_pulse,
     kernel_data,
     kernel_dna,
+    ml_anomalies,
     nginx_files,
     process_kernel_map,
     sentry_test,
@@ -63,6 +64,7 @@
     "isolation_context",
     "kernel_data",
     "kernel_dna",
+    "ml_anomalies",
     "network_stack_realtime",
     "nginx_files",
     "process_kernel_map",

kernel_ai/http/api_handlers/kernel.py

@@ -82,6 +82,41 @@ def kernel_dna():
     return api_json(_telemetry.get_kernel_dna_data)
 
 
+def ml_anomalies():
+    """Recent ML-detected anomalies (Stage 1 baselines) for Kernel DNA.
+
+    Read-only: the Flask app never computes anomalies, it only reads what the
+    isolated ML worker wrote to the shared store. If the store is unreachable,
+    the underlying reader returns an empty list rather than failing the page.
+    """
+
+    def _payload():
+        from kernel_ai.ml.config import MLConfig
+        from kernel_ai.ml.store import fetch_recent_anomalies
+
+        try:
+            since = int(request.args.get("since_seconds", 120))
+        except (TypeError, ValueError):
+            since = 120
+        since = max(5, min(since, 3600))
+        try:
+            limit = int(request.args.get("limit", 100))
+        except (TypeError, ValueError):
+            limit = 100
+        limit = max(1, min(limit, 500))
+
+        cfg = MLConfig()
+        anomalies = fetch_recent_anomalies(cfg.dsn, since_seconds=since, limit=limit)
+        return {
+            "timestamp": datetime.now().isoformat(),
+            "since_seconds": since,
+            "count": len(anomalies),
+            "anomalies": anomalies,
+        }
+
+    return api_json(_payload)
+
+
 def sentry_test():
     """Temporary endpoint for manual Sentry verification."""
     if not current_app.config.get("SENTRY_TEST_ENDPOINT_ENABLED", False):

kernel_ai/ml/__init__.py

@@ -0,0 +1,19 @@
+"""Kernel-AI anomaly detection pipeline (Stage 1: statistical baselines).
+
+This package is intentionally decoupled from the Flask request path:
+
+    collect features -> update baseline -> score -> persist anomalies
+
+The long-running detector lives in :mod:`kernel_ai.ml.worker` and runs as a
+separate process (``python -m kernel_ai.ml``). The Flask app only *reads*
+already-computed anomalies from the shared store, so a fault in the ML side can
+never block or crash the main service.
+
+Stages (see docs/KERNEL_DNA_AIML_ROADMAP.md):
+    Stage 1 (this code) - online EWMA baselines + robust z-score, pure-Python.
+    Stage 2+            - scikit-learn models, MLflow tracking (added later).
+"""
+
+from kernel_ai.ml.config import MLConfig
+
+__all__ = ["MLConfig"]

kernel_ai/ml/__main__.py

@@ -0,0 +1,6 @@
+"""Entrypoint: ``python -m kernel_ai.ml`` runs the Stage 1 detector loop."""
+
+from kernel_ai.ml.worker import main
+
+if __name__ == "__main__":
+    main()

kernel_ai/ml/baseline.py

@@ -0,0 +1,91 @@
+"""Stage 1 detector: online EWMA baseline + robust z-score.
+
+For every feature we keep a slowly-adapting estimate of its *normal* value
+(EWMA mean) and its *normal* spread (EWMA variance, West's incremental form).
+A live value far above the baseline (high positive z-score) is what we treat as
+an attack-shaped mutation: a sudden burst of processes, syscalls, retransmits,
+page faults, IRQs, etc.
+
+Pure Python on purpose — O(1) memory per feature, no numpy/sklearn — so it runs
+comfortably inside a tight memory budget. Heavier models arrive at Stage 2.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+
+@dataclass
+class _Stat:
+    mean: float = 0.0
+    var: float = 0.0
+    count: int = 0
+
+
+@dataclass(frozen=True)
+class Score:
+    name: str
+    value: float
+    mean: float
+    std: float
+    z: float
+    warm: bool
+
+
+class EwmaBaseline:
+    """Per-feature exponentially-weighted mean/variance with z-score scoring."""
+
+    def __init__(self, alpha: float, warmup_samples: int) -> None:
+        self.alpha = alpha
+        self.warmup = warmup_samples
+        self._stats: dict[str, _Stat] = {}
+
+    def _update(self, st: _Stat, x: float) -> None:
+        if st.count == 0:
+            st.mean = x
+            st.var = 0.0
+        else:
+            diff = x - st.mean
+            incr = self.alpha * diff
+            st.mean += incr
+            # West's incremental EWMA variance.
+            st.var = (1.0 - self.alpha) * (st.var + diff * incr)
+        st.count += 1
+
+    def update_and_score(self, features: dict[str, float], min_std: dict[str, float]) -> dict[str, Score]:
+        """Score the current values against the baseline, *then* fold them in.
+
+        Scoring before updating means a real spike is measured against the
+        pre-spike baseline (it hasn't yet been absorbed), which is what we want.
+        """
+        scores: dict[str, Score] = {}
+        for name, value in features.items():
+            st = self._stats.setdefault(name, _Stat())
+            warm = st.count < self.warmup
+            std = math.sqrt(max(0.0, st.var))
+            floor = max(min_std.get(name, 0.0), 1e-9)
+            std_eff = max(std, floor)
+            z = (value - st.mean) / std_eff if st.count > 0 else 0.0
+            scores[name] = Score(name=name, value=value, mean=st.mean, std=std, z=z, warm=warm)
+            self._update(st, value)
+        return scores
+
+    # --- warm-restart support: snapshot / restore baseline across restarts ---
+
+    def export_state(self) -> list[dict]:
+        return [
+            {"name": n, "mean": s.mean, "var": s.var, "count": s.count}
+            for n, s in self._stats.items()
+        ]
+
+    def load_state(self, rows: list[dict]) -> None:
+        for row in rows or []:
+            try:
+                self._stats[row["name"]] = _Stat(
+                    mean=float(row["mean"]),
+                    var=float(row["var"]),
+                    count=int(row["count"]),
+                )
+            except (KeyError, TypeError, ValueError):
+                continue

kernel_ai/ml/config.py

@@ -0,0 +1,64 @@
+"""Configuration for the ML anomaly pipeline.
+
+Everything is driven by environment variables so the same code runs locally
+(local Postgres) and on PROD (managed Postgres) by only swapping the DSN.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.getenv(name, str(default)))
+    except (TypeError, ValueError):
+        return default
+
+
+def _env_int(name: str, default: int) -> int:
+    try:
+        return int(os.getenv(name, str(default)))
+    except (TypeError, ValueError):
+        return default
+
+
+@dataclass(frozen=True)
+class MLConfig:
+    """Resolved ML pipeline settings (read once at process start)."""
+
+    # Postgres connection string. On deploy, override with the PROD value.
+    # Example: postgresql://user:pass@host:5432/dbname
+    dsn: str = os.getenv(
+        "KERNEL_AI_ML_DSN",
+        "postgresql://kernel_ai:kernel_ai_dev_pw@127.0.0.1:5432/kernel_ai_ml",
+    )
+
+    # Sampling cadence of the detector loop (seconds between feature snapshots).
+    interval_sec: float = _env_float("KERNEL_AI_ML_INTERVAL_SEC", 2.0)
+
+    # EWMA smoothing: alpha = 2 / (window + 1). Larger window = slower, calmer
+    # baseline. ~60 gives a baseline that adapts over a couple of minutes.
+    baseline_window: int = _env_int("KERNEL_AI_ML_BASELINE_WINDOW", 60)
+
+    # Samples to observe before we start emitting anomalies (let the baseline
+    # settle so we don't fire on the cold-start transient).
+    warmup_samples: int = _env_int("KERNEL_AI_ML_WARMUP", 30)
+
+    # Robust z-score thresholds. A feature whose current value sits this many
+    # standard deviations above its baseline becomes a mutation.
+    z_warn: float = _env_float("KERNEL_AI_ML_Z_WARN", 4.0)
+    z_crit: float = _env_float("KERNEL_AI_ML_Z_CRIT", 7.0)
+
+    # Persist the raw feature snapshot each tick (useful for later training /
+    # drift analysis). Disable to keep the DB tiny.
+    store_features: bool = os.getenv("KERNEL_AI_ML_STORE_FEATURES", "true").lower() == "true"
+
+    # How long to keep rows (hours). The worker prunes older data each cycle.
+    retain_features_hours: int = _env_int("KERNEL_AI_ML_RETAIN_FEATURES_H", 48)
+    retain_anomalies_hours: int = _env_int("KERNEL_AI_ML_RETAIN_ANOMALIES_H", 168)
+
+    @property
+    def alpha(self) -> float:
+        return 2.0 / (max(2, self.baseline_window) + 1.0)