Skip to content

service communication channels + filesystem #126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
devops-stack merged 1 commit into from
Jun 15, 2026
Merged

service communication channels + filesystem #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">

<!-- Styles -->
<link rel="stylesheet" href="/static/css/main.css?v=3">
<link rel="stylesheet" href="/static/css/main.css?v=4">
</head>
<body>
<!-- ================= SEO CONTENT (INDEXED BY GOOGLE) ================= -->
Expand Down Expand Up @@ -84,13 +84,13 @@ <h2>Linux Kernel Ring 0 Visualization</h2>
<script src="/static/js/syscalls.js?v=12"></script>
<script src="/static/js/subsystem-focus.js?v=1"></script>
<script src="/static/js/irq-ui.js?v=1"></script>
<script src="/static/js/ipc-ui.js?v=1"></script>
<script src="/static/js/flow-ui.js?v=1"></script>
<script src="/static/js/ipc-ui.js?v=2"></script>
<script src="/static/js/flow-ui.js?v=6"></script>
<script src="/static/js/isolation-ui.js?v=1"></script>
<script src="/static/js/process-files-ui.js?v=1"></script>
<script src="/static/js/ui-chrome.js?v=1"></script>
<script src="/static/js/active-connections.js?v=4"></script>
<script src="/static/js/nginx_files.js?v=2"></script>
<script src="/static/js/nginx_files.js?v=5"></script>
<script src="/static/js/right-semicircle-menu.js?v=48"></script>
<script src="/static/js/kernel-dna.js?v=31"></script>
<script src="/static/js/network-stack.js?v=6"></script>
Expand All @@ -108,6 +108,6 @@ <h2>Linux Kernel Ring 0 Visualization</h2>
console.error('❌ RightSemicircleMenuManager class NOT loaded!');
}
</script>
<script src="/static/js/main.js?v=172"></script>
<script src="/static/js/main.js?v=192"></script>
</body>
</html>
1 change: 1 addition & 0 deletions kernel_ai/api/rest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
("/process-kernel-map", "process_kernel_map", h.process_kernel_map, None),
("/processes", "get_processes", h.get_processes, None),
("/nginx-files", "nginx_files", h.nginx_files, None),
("/io-open-files", "io_open_files", h.io_open_files, None),
("/active-connections", "active_connections", h.active_connections, None),
("/traceroute", "traceroute_info", h.traceroute_info, None),
("/network-stack-realtime", "network_stack_realtime", h.network_stack_realtime, None),
Expand Down
2 changes: 2 additions & 0 deletions kernel_ai/http/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from kernel_ai.http.api_handlers.kernel import (
get_execution_context,
io_open_files,
kernel_data,
kernel_dna,
nginx_files,
Expand Down Expand Up @@ -56,6 +57,7 @@
"get_processes",
"get_processes_detailed",
"ingest_frontend_logs",
"io_open_files",
"isolation_context",
"kernel_data",
"kernel_dna",
Expand Down
12 changes: 12 additions & 0 deletions kernel_ai/http/api_handlers/kernel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ def nginx_files():
return api_json(lambda: {"files": _telemetry.get_nginx_open_files()})


def io_open_files():
def _payload():
try:
limit = int(request.args.get("limit", 40))
except (TypeError, ValueError):
limit = 40
limit = max(1, min(limit, 80))
return {"files": _telemetry.get_io_open_files(limit=limit)}

return api_json(_payload)


def get_execution_context():
return api_json(
lambda: _telemetry.get_execution_context_data(
Expand Down
107 changes: 107 additions & 0 deletions kernel_ai/services/core_observability.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,113 @@ def get_mock_nginx_files():
]


def _classify_io_file_path(path):
"""Classify an open-file path into a coarse filesystem category."""
low = path.lower()
if "/var/log/" in low or low.endswith(".log"):
return "log"
if "/etc/" in low or low.endswith(
(".conf", ".cfg", ".ini", ".yaml", ".yml", ".json", ".toml")
):
return "config"
if low.endswith(".so") or ".so." in low or "/lib/" in low or "/lib64/" in low:
return "lib"
if "/dev/" in low:
return "device"
if low.endswith((".db", ".sqlite", ".sqlite3")) or "/var/lib/" in low:
return "data"
return "other"


def _io_open_files_mock():
"""Mock data for the system-wide open-files I/O layer."""
return [
{"path": "/lib/x86_64-linux-gnu/libc.so.6", "type": "lib", "activity": 14,
"process": "gunicorn", "process_count": 9, "pids": []},
{"path": "/etc/nginx/nginx.conf", "type": "config", "activity": 6,
"process": "nginx", "process_count": 3, "pids": []},
{"path": "/var/log/nginx/access.log", "type": "log", "activity": 5,
"process": "nginx", "process_count": 2, "pids": []},
{"path": "/var/lib/postgresql/data/base", "type": "data", "activity": 4,
"process": "postgres", "process_count": 2, "pids": []},
{"path": "/var/log/syslog", "type": "log", "activity": 3,
"process": "rsyslogd", "process_count": 1, "pids": []},
{"path": "/etc/ssl/certs/ca-certificates.crt", "type": "config", "activity": 2,
"process": "python3", "process_count": 1, "pids": []},
]


def get_io_open_files(limit=40, max_procs=600):
"""Aggregate open files across processes ranked by how widely they're held.

"Activity" is approximated by the number of open handles to a path across the
system, which surfaces hot/shared files (libs, configs, logs) for the
KERNEL I/O LAYER visualization. Returns a list sorted by activity desc.
"""
try:
counts = {}
scanned = 0
for proc in psutil.process_iter(["pid", "name"]):
if scanned >= max_procs:
break
scanned += 1
try:
open_files = proc.open_files()
except (psutil.NoSuchProcess, psutil.AccessDenied, OSError):
continue
name = proc.info.get("name") or ""
pid = proc.info.get("pid")
for file in open_files:
path = getattr(file, "path", None)
if not path:
continue
rec = counts.get(path)
if rec is None:
rec = {
"path": path,
"type": _classify_io_file_path(path),
"count": 0,
"procs": set(),
"pids": set(),
}
counts[path] = rec
rec["count"] += 1
if name:
rec["procs"].add(name)
if pid is not None:
rec["pids"].add(pid)

if not counts:
return _io_open_files_mock()

ranked = sorted(counts.values(), key=lambda r: r["count"], reverse=True)[:limit]
result = []
for rec in ranked:
procs = sorted(rec["procs"])
result.append(
{
"path": rec["path"],
"type": rec["type"],
"activity": rec["count"],
"process": procs[0] if procs else "",
"process_count": len(procs),
"pids": sorted(rec["pids"])[:32],
}
)
return result
except (psutil.Error, OSError, ValueError) as exc:
log_event(
logger,
"DEBUG",
"Failed to aggregate system open files, using mock",
event_dataset="kernel_ai.app",
component="services.core_observability",
operation="get_io_open_files",
event_data={"error": str(exc)},
)
return _io_open_files_mock()


def get_nginx_open_files():
"""Get open files for Nginx process."""
try:
Expand Down
Loading
Loading