devops-stack · devops-stack · Aug 14, 2026 · Aug 14, 2026
diff --git a/gunicorn.conf.py b/gunicorn.conf.py
@@ -4,8 +4,16 @@
 #   rm -rf "$PROMETHEUS_MULTIPROC_DIR" && mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
 
 
+import os
+
+
 def child_exit(server, worker):
     """Required for prometheus_client multiprocess mode (gunicorn -w N > 1)."""
+    # Without the directory there are no per-worker metric files to clean up, and
+    # prometheus_client raises TypeError on the None path rather than saying so —
+    # once per worker exit, which is every restart.
+    if not os.environ.get("PROMETHEUS_MULTIPROC_DIR", "").strip():
+        return
     try:
         from prometheus_client import multiprocess
 

diff --git a/index.html b/index.html
@@ -37,7 +37,7 @@
     <link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
 
     <!-- Styles -->
-    <link rel="stylesheet" href="/static/css/main.css?v=13">
+    <link rel="stylesheet" href="/static/css/main.css?v=20">
 </head>
 <body>
     <!-- ================= SEO CONTENT (INDEXED BY GOOGLE) ================= -->
@@ -87,19 +87,26 @@ <h2>Linux Kernel Ring 0 Visualization</h2>
 <!-- ================= SCRIPTS ================= -->
     <script src="/static/js/safe-utils.js?v=2"></script>
     <script src="/static/js/frontend-logger.js?v=2"></script>
-    <script src="/static/js/syscalls.js?v=12"></script>
-    <script src="/static/js/subsystem-focus.js?v=1"></script>
-    <script src="/static/js/irq-ui.js?v=1"></script>
+    <script src="/static/js/syscalls.js?v=16"></script>
+    <script src="/static/js/syscall-card.js?v=2"></script>
+    <script src="/static/js/runqueue-card.js?v=1"></script>
+    <script src="/static/js/wakeups-card.js?v=1"></script>
+    <script src="/static/js/subsystem-focus.js?v=6"></script>
+    <script src="/static/js/irq-ui.js?v=7"></script>
+    <script src="/static/js/irq-card.js?v=4"></script>
+    <script src="/static/js/threads-card.js?v=3"></script>
+    <script src="/static/js/memory-card.js?v=4"></script>
+    <script src="/static/js/waits-card.js?v=2"></script>
     <script src="/static/js/ipc-ui.js?v=4"></script>
     <script src="/static/js/flow-ui.js?v=8"></script>
-    <script src="/static/js/isolation-ui.js?v=10"></script>
+    <script src="/static/js/isolation-ui.js?v=11"></script>
     <script src="/static/js/process-files-ui.js?v=1"></script>
     <script src="/static/js/ui-chrome.js?v=2"></script>
     <script src="/static/js/active-connections.js?v=5"></script>
     <script src="/static/js/nginx_files.js?v=6"></script>
-    <script src="/static/js/right-semicircle-menu.js?v=51"></script>
+    <script src="/static/js/right-semicircle-menu.js?v=53"></script>
     <script src="/static/js/kernel-dna.js?v=49"></script>
-    <script src="/static/js/network-stack.js?v=41"></script>
+    <script src="/static/js/network-stack.js?v=61"></script>
     <script src="/static/js/devices-belt.js?v=25"></script>
     <script src="/static/js/aes-ref.js?v=3"></script>
     <script src="/static/js/crypto-belt.js?v=61"></script>
@@ -115,7 +122,7 @@ <h2>Linux Kernel Ring 0 Visualization</h2>
             console.error('❌ RightSemicircleMenuManager class NOT loaded!');
         }
     </script>
-    <script src="/static/js/main.js?v=204"></script>
-    <script src="/static/js/kernel-tape.js?v=4"></script>
+    <script src="/static/js/main.js?v=232"></script>
+    <script src="/static/js/kernel-tape.js?v=13"></script>
 </body>
 </html>
diff --git a/kernel_ai/api/rest.py b/kernel_ai/api/rest.py
@@ -25,6 +25,8 @@
     ("/path-walk", "path_walk", h.path_walk, None),
     ("/isolation-context", "isolation_context", h.isolation_context, None),
     ("/process/<int:pid>/threads", "get_process_threads", h.get_process_threads, None),
+    ("/process/<int:pid>/memory", "get_process_memory", h.get_process_memory, None),
+    ("/process/<int:pid>/thread/<int:tid>/wait", "get_thread_wait", h.get_thread_wait, None),
     ("/process/<int:pid>/cpu", "get_process_cpu", h.get_process_cpu, None),
     ("/process/<int:pid>/fds", "get_process_fds", h.get_process_fds, None),
     ("/process/<int:pid>/lineage", "get_process_lineage", h.get_process_lineage, None),
@@ -44,6 +46,8 @@
     ("/security-realtime", "security_realtime", h.security_realtime, None),
     ("/processes-realtime", "processes_realtime", h.processes_realtime, None),
     ("/scheduler-pelt", "scheduler_pelt", h.scheduler_pelt, None),
+    ("/runqueue", "runqueue", h.runqueue, None),
+    ("/wakeups", "wakeups", h.wakeups, None),
     ("/frontend-logs", "ingest_frontend_logs", h.ingest_frontend_logs, ["POST", "OPTIONS"]),
     ("/proc-graph", "proc_graph", h.get_proc_graph, None),
     ("/process-files", "process_files", h.get_process_files, None),

diff --git a/kernel_ai/http/api.py b/kernel_ai/http/api.py
@@ -11,10 +11,12 @@
     ml_drift,
     nginx_files,
     process_kernel_map,
+    runqueue,
     sentry_test,
     siem_alerts,
     syscall_detail,
     syscalls_realtime,
+    wakeups,
 )
 from kernel_ai.http.api_handlers.network_system import (
     active_connections,
@@ -39,7 +41,9 @@
     get_process_activity,
     get_process_files,
     get_process_lineage,
+    get_process_memory,
     get_process_threads,
+    get_thread_wait,
     get_processes,
     get_processes_detailed,
     processes_realtime,
@@ -74,7 +78,9 @@
     "get_process_activity",
     "get_process_kernel_map",
     "get_process_lineage",
+    "get_process_memory",
     "get_process_threads",
+    "get_thread_wait",
     "get_processes",
     "get_processes_detailed",
     "ingest_frontend_logs",
@@ -90,12 +96,14 @@
     "nginx_files",
     "process_kernel_map",
     "processes_realtime",
+    "runqueue",
     "scheduler_pelt",
     "security_realtime",
     "sentry_test",
     "siem_alerts",
     "syscall_detail",
     "syscalls_realtime",
     "traceroute_info",
+    "wakeups",
 ]
 
diff --git a/kernel_ai/http/api_handlers/kernel.py b/kernel_ai/http/api_handlers/kernel.py
@@ -14,15 +14,108 @@
 
 
 def syscalls_realtime():
-    return api_json(
-        lambda: {
+    def _payload():
+        sample = _telemetry.get_syscall_sample()
+        return {
             "timestamp": datetime.now().isoformat(),
-            "syscalls": _telemetry.get_real_system_calls(),
+            "syscalls": sample.get("syscalls", []),
+            # How the sample was taken. "self" means the root collector is not
+            # running and the panel can only see the backend's own processes,
+            # which the UI has to say out loud rather than pass off as the machine.
+            "sample": {
+                "source": sample.get("source"),
+                "scope": sample.get("scope"),
+                "tasks_total": sample.get("tasks_total"),
+                "blocked_total": sample.get("blocked_total"),
+                "age": sample.get("age"),
+            },
             "cpu_usage": psutil.cpu_percent(interval=1),
             "memory_usage": psutil.virtual_memory().percent,
             "system_info": _core_observability_service.get_system_info(),
         }
-    )
+
+    return api_json(_payload)
+
+
+def syscall_detail(name):
+    """What one syscall is inside this kernel, plus who is parked in it.
+
+    The number, the chain of kernel symbols and the sleeping function all come
+    from the running machine. A call nobody is parked in right now still has an
+    answer — the waiter list is simply empty.
+    """
+
+    def _payload():
+        from collections import Counter
+
+        from kernel_ai.services import syscall_anatomy
+
+        clean = "".join(ch for ch in str(name)[:64] if ch.isalnum() or ch == "_")
+        sample = _telemetry.get_syscall_sample()
+        rows = sample.get("syscalls") or []
+        row = next((r for r in rows if str(r.get("name", "")).lower() == clean.lower()), None)
+
+        waiters = list(row.get("waiters") or []) if row else []
+        numbers = {v: k for k, v in _telemetry.get_syscall_names().items()}
+        nr = (row or {}).get("nr")
+        if nr is None:
+            nr = numbers.get(clean)
+        subsystem = (row or {}).get("subsystem") or _telemetry.map_syscall_to_subsystem(clean)
+
+        wchans = Counter(w.get("wchan") for w in waiters if w.get("wchan"))
+        anatomy = syscall_anatomy.describe(
+            clean, nr=nr, subsystem=subsystem, wchans=wchans.most_common(), sampled=len(waiters)
+        )
+
+        return {
+            "timestamp": datetime.now().isoformat(),
+            **anatomy,
+            "count": (row or {}).get("count", 0),
+            "waiters": waiters,
+            "sample": {"source": sample.get("source"), "scope": sample.get("scope")},
+        }
+
+    return api_json(_payload)
+
+
+def wakeups():
+    """Who woke whom, over the window the collector last sampled."""
+
+    def _payload():
+        from kernel_ai.services import wakeups as wakeups_service
+        return {"timestamp": datetime.now().isoformat(), **wakeups_service.describe()}
+    return api_json(_payload)
+
+
+def runqueue():
+    """Who is competing for a CPU right now, and who would be taken next."""
+
+    def _payload():
+        from kernel_ai.services import runqueue as runqueue_service
+
+        return {"timestamp": datetime.now().isoformat(), **runqueue_service.describe()}
+
+    return api_json(_payload)
+
+
+def irq_detail(irq):
+    """What one interrupt line is on this machine.
+
+    Identity, affinity, the per-CPU counters and the deferred half, all read
+    from /sys and /proc. Rates are not part of the answer: the panel measures
+    them over its polling interval, which is a steadier window than a single
+    request could sample, and the card uses those.
+    """
+
+    def _payload():
+        from kernel_ai.services import irq_anatomy
+
+        anatomy = irq_anatomy.describe(irq)
+        if anatomy is None:
+            return {"timestamp": datetime.now().isoformat(), "irq": str(irq), "found": False}
+        return {"timestamp": datetime.now().isoformat(), "found": True, **anatomy}
+
+    return api_json(_payload)
 
 
 def io_pulse():

diff --git a/kernel_ai/http/api_handlers/processes.py b/kernel_ai/http/api_handlers/processes.py
@@ -8,14 +8,25 @@
 from kernel_ai.services import process_timeline as _process_timeline_service
 from kernel_ai.services import processes as _processes_service
 from kernel_ai.services import scheduler_pelt as _scheduler_pelt_service
+from kernel_ai.services import memory as _memory_service
+from kernel_ai.services import threads as _threads_service
+from kernel_ai.services import waits as _waits_service
 
 
 def get_processes():
     return api_json(lambda: {"processes": _processes_service.get_processes_basic_data()})
 
 
 def get_process_threads(pid):
-    return api_json(lambda: _process_inspect_service.get_process_threads_info(pid))
+    return api_json(lambda: _threads_service.describe(pid))
+
+
+def get_process_memory(pid):
+    return api_json(lambda: _memory_service.describe(pid))
+
+
+def get_thread_wait(pid, tid):
+    return api_json(lambda: _waits_service.describe(pid, tid))
 
 
 def get_process_cpu(pid):

diff --git a/kernel_ai/services/execution.py b/kernel_ai/services/execution.py
@@ -12,6 +12,61 @@
 from kernel_ai.sentry_helpers import capture_exception
 
 
+_IRQ_DEVICE_CACHE: dict = {}
+
+
+def _irq_vector(device: str, desc: str):
+    from kernel_ai.services import irq_anatomy
+
+    return irq_anatomy.vector_for(device or desc, desc)
+
+
+def _softirq_symbol(vector: str):
+    """The kernel function a softirq vector runs, if this kernel exports it."""
+    from kernel_ai.services import irq_anatomy
+
+    symbol = irq_anatomy.VECTOR_SYMBOL.get(str(vector).upper())
+    if not symbol:
+        return None
+    symbols = irq_anatomy.kernel_symbols()
+    return symbol if (symbols and symbol in symbols) else None
+
+
+def _irq_device(irq: str) -> str:
+    """The handler name the driver registered for a numbered line.
+
+    /proc/interrupts prints the chip and the trigger before the device, so the
+    raw description truncates to something useless like "xen-dyn-latee…" in a
+    narrow panel. /sys/kernel/irq/<n>/actions gives the device on its own.
+    A line's handler is fixed once the driver has probed, so it is read once.
+    """
+    if irq in _IRQ_DEVICE_CACHE:
+        return _IRQ_DEVICE_CACHE[irq]
+    device = ""
+    if irq.isdigit():
+        try:
+            with open(f"/sys/kernel/irq/{irq}/actions", "r", encoding="utf-8", errors="replace") as fh:
+                device = fh.read().strip()
+        except OSError:
+            device = ""
+    _IRQ_DEVICE_CACHE[irq] = device
+    return device
+
+
+def _irq_short_label(desc: str) -> str:
+    """The kernel's description of a lettered counter, minus the obvious word.
+
+    Those rows read "Hypervisor callback interrupts" and "Machine check polls";
+    under a panel already titled INTERRUPTS the last word is a waste of the
+    twenty characters the column has.
+    """
+    text = str(desc or "").strip()
+    for tail in (" interrupts", " polls", " events"):
+        if text.lower().endswith(tail):
+            return text[: -len(tail)]
+    return text
+
+
 def _read_key_value_proc_file(path: str) -> dict:
     out = {}
     try:
@@ -332,14 +387,22 @@ def get_execution_context_data(syscall_names, map_interrupt_to_subsystem_fn, exe
                 per_sec = max(0.0, (total - prev_total) / dt)
 
             top_cpu = int(max(range(len(counts)), key=lambda i: counts[i])) if counts else None
+            device = _irq_device(irq_name)
+            vector = _irq_vector(device, desc)
             irq_rows.append(
                 {
                     "irq": irq_name,
                     "label": desc,
+                    # What the panel prints: the device, when the driver named
+                    # one, and otherwise the kernel's own description.
+                    "device": device or _irq_short_label(desc),
                     "total": int(total),
                     "per_sec": round(per_sec, 2),
                     "top_cpu": top_cpu,
-                    "subsystem": map_interrupt_to_subsystem_fn(desc),
+                    "subsystem": map_interrupt_to_subsystem_fn(device or desc),
+                    # Concluded from the class of the device, never measured;
+                    # the route map that shows it says so on the same line.
+                    "vector": vector,
                 }
             )
     except (IOError, PermissionError):
@@ -367,7 +430,16 @@ def get_execution_context_data(syscall_names, map_interrupt_to_subsystem_fn, exe
             per_sec = 0.0
             if dt and prev_total is not None:
                 per_sec = max(0.0, (total - prev_total) / dt)
-            softirq_rows.append({"name": name, "total": int(total), "per_sec": round(per_sec, 2)})
+            softirq_rows.append(
+                {
+                    "name": name,
+                    "total": int(total),
+                    "per_sec": round(per_sec, 2),
+                    # The function this vector runs, so the route map can end on
+                    # a real kernel symbol instead of a placeholder.
+                    "symbol": _softirq_symbol(name),
+                }
+            )
     except (IOError, PermissionError):
         pass