diff --git a/Containerfile.c10s b/Containerfile.c10s index 6545a3184..51214b686 100644 --- a/Containerfile.c10s +++ b/Containerfile.c10s @@ -69,6 +69,7 @@ RUN dnf -y install --allowerasing \ COPY beeai-reasoning.patch /tmp COPY openinference-reasoning.patch /tmp +COPY openinference-streaming.patch /tmp RUN pip3 install --no-cache-dir \ "litellm!=1.82.7,!=1.82.8,!=1.92.0" \ @@ -85,7 +86,8 @@ RUN pip3 install --no-cache-dir \ sentry-sdk>=2.13.0 \ && cd /usr/local/lib/python3.12/site-packages \ && patch -p2 -i /tmp/beeai-reasoning.patch \ - && patch -p5 -i /tmp/openinference-reasoning.patch + && patch -p5 -i /tmp/openinference-reasoning.patch \ + && patch -p5 -i /tmp/openinference-streaming.patch # Verify no malicious litellm_init.pth was introduced by compromised litellm packages (e.g. 1.82.7, 1.82.8) RUN MALICIOUS=$(find /usr /opt -name "litellm_init.pth" 2>/dev/null); \ diff --git a/Containerfile.c9s b/Containerfile.c9s index 8d6a2d452..5f36e353d 100644 --- a/Containerfile.c9s +++ b/Containerfile.c9s @@ -69,6 +69,7 @@ RUN dnf -y install --allowerasing \ COPY beeai-reasoning.patch /tmp COPY openinference-reasoning.patch /tmp +COPY openinference-streaming.patch /tmp # Create Python 3.11 virtual environment and install Python packages RUN python3.11 -m venv --system-site-packages /opt/beeai-venv \ @@ -87,7 +88,8 @@ RUN python3.11 -m venv --system-site-packages /opt/beeai-venv \ sentry-sdk>=2.13.0 \ && cd /opt/beeai-venv/lib/python3.11/site-packages \ && patch -p2 -i /tmp/beeai-reasoning.patch \ - && patch -p5 -i /tmp/openinference-reasoning.patch + && patch -p5 -i /tmp/openinference-reasoning.patch \ + && patch -p5 -i /tmp/openinference-streaming.patch # Verify no malicious litellm_init.pth was introduced by compromised litellm packages (e.g. 1.82.7, 1.82.8) RUN MALICIOUS=$(find /usr /opt -name "litellm_init.pth" 2>/dev/null); \ diff --git a/openinference-streaming.patch b/openinference-streaming.patch new file mode 100644 index 000000000..84f8c42b7 --- /dev/null +++ b/openinference-streaming.patch @@ -0,0 +1,167 @@ +diff --git a/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/__init__.py b/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/__init__.py +index d3f3adc6..5497ff77 100644 +--- a/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/__init__.py ++++ b/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/__init__.py +@@ -1,18 +1,15 @@ +-import contextlib + import logging + from importlib.metadata import PackageNotFoundError, version +-from typing import TYPE_CHECKING, Any, Callable, Collection, Generator ++from typing import TYPE_CHECKING, Any, Callable, Collection + ++from opentelemetry import context as context_api ++from opentelemetry import trace as trace_api ++from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore + from opentelemetry.trace import StatusCode + +-from openinference.instrumentation._spans import OpenInferenceSpan +- + if TYPE_CHECKING: + from beeai_framework.emitter import EventMeta + +-from opentelemetry import trace as trace_api +-from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore +- + from openinference.instrumentation import ( + OITracer, + TraceConfig, +@@ -32,13 +29,14 @@ except PackageNotFoundError: + + + class BeeAIInstrumentor(BaseInstrumentor): # type: ignore +- __slots__ = ("_tracer", "_cleanup", "_processes", "_processes_deps") ++ __slots__ = ("_tracer", "_cleanup", "_processes", "_otel_spans", "_otel_contexts") + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._cleanup: Callable[[], None] = lambda: None + self._processes: dict[str, Processor] = {} +- self._processes_deps: dict[str, list[Processor]] = {} ++ self._otel_spans: dict[str, trace_api.Span] = {} ++ self._otel_contexts: dict[str, context_api.Context] = {} + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments +@@ -70,39 +68,76 @@ class BeeAIInstrumentor(BaseInstrumentor): # type: ignore + def _uninstrument(self, **kwargs: Any) -> None: + self._cleanup() + self._processes.clear() +- self._processes_deps.clear() ++ self._otel_spans.clear() ++ self._otel_contexts.clear() ++ ++ def _start_otel_span(self, processor: Processor, parent_run_id: str | None) -> None: ++ parent_ctx = None ++ if parent_run_id and parent_run_id in self._otel_contexts: ++ parent_ctx = self._otel_contexts[parent_run_id] ++ ++ span = self._tracer.start_span( ++ name=processor.span.name, ++ openinference_span_kind=processor.span.kind, ++ attributes=dict(processor.span.attributes), ++ start_time=_datetime_to_span_time(processor.span.started_at) if processor.span.started_at else None, ++ context=parent_ctx, ++ ) ++ ++ ctx = trace_api.set_span_in_context(span, parent_ctx or context_api.get_current()) ++ self._otel_spans[processor.run_id] = span ++ self._otel_contexts[processor.run_id] = ctx ++ ++ def _end_otel_span(self, processor: Processor) -> None: ++ span = self._otel_spans.pop(processor.run_id, None) ++ if span is None: ++ self._otel_contexts.pop(processor.run_id, None) ++ return + +- def _build_tree(self, processor: Processor) -> None: +- with self._build_tree_for_span(processor.span): +- for child in self._processes_deps.pop(processor.run_id): +- self._build_tree(child) +- self._processes.pop(processor.run_id) ++ node = processor.span ++ for key, value in node.attributes.items(): ++ span.set_attribute(key, value) + +- @contextlib.contextmanager +- def _build_tree_for_span(self, node: SpanWrapper) -> Generator[OpenInferenceSpan, None, None]: +- with self._tracer.start_as_current_span( ++ for event in node.events: ++ span.add_event( ++ name=event.name, attributes=event.attributes, timestamp=event.timestamp ++ ) ++ ++ parent_ctx = self._otel_contexts.get(processor.run_id) ++ for child in node.children: ++ self._build_inline_child(child, parent_ctx) ++ ++ self._otel_contexts.pop(processor.run_id, None) ++ ++ span.set_status(node.status) ++ if node.error is not None and node.status == StatusCode.ERROR: ++ span.record_exception(node.error) ++ ++ span.end(_datetime_to_span_time(node.ended_at) if node.ended_at else None) ++ ++ def _build_inline_child(self, node: SpanWrapper, parent_ctx: context_api.Context | None) -> None: ++ child_span = self._tracer.start_span( + name=node.name, + openinference_span_kind=node.kind, + attributes=node.attributes, + start_time=_datetime_to_span_time(node.started_at) if node.started_at else None, +- end_on_exit=False, # we do it manually +- ) as current_span: +- yield current_span ++ context=parent_ctx, ++ ) + +- for event in node.events: +- current_span.add_event( +- name=event.name, attributes=event.attributes, timestamp=event.timestamp +- ) ++ for event in node.events: ++ child_span.add_event( ++ name=event.name, attributes=event.attributes, timestamp=event.timestamp ++ ) + +- for children in node.children: +- with self._build_tree_for_span(children): +- pass ++ child_ctx = trace_api.set_span_in_context(child_span, parent_ctx or context_api.get_current()) ++ for descendant in node.children: ++ self._build_inline_child(descendant, child_ctx) + +- current_span.set_status(node.status) +- if node.error is not None and node.status == StatusCode.ERROR: +- current_span.record_exception(node.error) ++ child_span.set_status(node.status) ++ if node.error is not None and node.status == StatusCode.ERROR: ++ child_span.record_exception(node.error) + +- current_span.end(_datetime_to_span_time(node.ended_at) if node.ended_at else None) ++ child_span.end(_datetime_to_span_time(node.ended_at) if node.ended_at else None) + + @exception_handler + async def _handler(self, data: Any, event: "EventMeta") -> None: +@@ -118,10 +153,8 @@ class BeeAIInstrumentor(BaseInstrumentor): # type: ignore + if event.trace.parent_run_id and not parent: + raise ValueError(f"Parent run with ID {event.trace.parent_run_id} was not found!") + +- self._processes_deps[event.trace.run_id] = [] + node = self._processes[event.trace.run_id] = ProcessorLocator.locate(data, event) +- if parent is not None: +- self._processes_deps[parent.run_id].append(node) ++ self._start_otel_span(node, event.trace.parent_run_id) + else: + node = self._processes[event.trace.run_id] + +@@ -129,8 +162,8 @@ class BeeAIInstrumentor(BaseInstrumentor): # type: ignore + + if isinstance(data, RunContextFinishEvent): + await node.end(data, event) +- if event.trace.parent_run_id is None: +- self._build_tree(node) ++ self._end_otel_span(node) ++ self._processes.pop(event.trace.run_id, None) + else: + if event.context.get("internal"): + return diff --git a/trace_server/renderer.py b/trace_server/renderer.py deleted file mode 100644 index 04c2cb1d8..000000000 --- a/trace_server/renderer.py +++ /dev/null @@ -1,261 +0,0 @@ -"""HTML rendering for the trace server.""" - -from datetime import UTC, datetime -from html import escape - - -def _get_val(value: dict): - if not isinstance(value, dict): - return None - for k in ("stringValue", "intValue", "boolValue", "doubleValue"): - if k in value: - return value[k] - return None - - -STATUS_LABELS = {0: "Unset", 1: "Ok", 2: "Error"} - -HTML_HEAD = """\ - -{title} - - -""" - -HTML_FOOT = """\ -""" - - -def _fmt_time(nanos: int | None) -> str: - if not nanos: - return "-" - return datetime.fromtimestamp(nanos / 1e9, tz=UTC).strftime("%Y-%m-%d %H:%M:%S") - - -def _fmt_duration(start: int, end: int | None) -> str: - if not end or not start: - return "-" - ms = (end - start) / 1e6 - if ms < 1000: - return f"{ms:.0f}ms" - return f"{ms / 1000:.1f}s" - - -def _status_class(code: int) -> str: - if code == 2: - return "status-error" - if code == 1: - return "status-ok" - return "" - - -def _extract_detail(attrs: dict, span_name: str = "") -> str | None: - span_kind = _get_val(attrs.get("openinference.span.kind")) - - if span_kind == "LLM": - parts = [] - i = 0 - while True: - ctype = _get_val(attrs.get(f"llm.output_messages.0.message.contents.{i}.message_content.type")) - if ctype is None: - break - if ctype == "reasoning": - text = _get_val(attrs.get(f"llm.output_messages.0.message.contents.{i}.message_content.text")) - if text: - parts.append(("reasoning", text)) - elif ctype == "text": - text = _get_val(attrs.get(f"llm.output_messages.0.message.contents.{i}.message_content.text")) - if text: - parts.append(("text", text)) - i += 1 - - i = 0 - while True: - name = _get_val( - attrs.get(f"llm.output_messages.0.message.tool_calls.{i}.tool_call.function.name") - ) - if name is None: - break - args = ( - _get_val( - attrs.get(f"llm.output_messages.0.message.tool_calls.{i}.tool_call.function.arguments") - ) - or "" - ) - parts.append(("tool_call", f"{name}({args})")) - i += 1 - - if not parts: - return None - - html = [] - for kind, content in parts: - if kind == "reasoning": - html.append(f'
{escape(content)}
') - elif kind == "text": - html.append(f'
{escape(content)}
') - elif kind == "tool_call": - html.append(f'
{escape(content)}
') - return "".join(html) - - if span_name == "error": - output_val = _get_val(attrs.get("output.value")) - if output_val: - output_str = str(output_val) - truncated = output_str[:500] + ("..." if len(output_str) > 500 else "") - return f'
{escape(truncated)}
' - return None - - if span_kind == "TOOL": - tool_name = _get_val(attrs.get("tool.name")) - input_val = _get_val(attrs.get("input.value")) - output_val = _get_val(attrs.get("output.value")) - if not tool_name: - return None - html = [f'
{escape(tool_name)}
'] - if input_val is not None: - input_str = str(input_val) - truncated = input_str[:500] + ("..." if len(input_str) > 500 else "") - html.append(f"
input
{escape(truncated)}
") - if output_val is not None: - output_str = str(output_val) - truncated = output_str[:500] + ("..." if len(output_str) > 500 else "") - html.append(f"
output
{escape(truncated)}
") - return "".join(html) - - return None - - -def _render_attrs(attrs: dict) -> str: - if not attrs: - return "" - rows = [] - for k, v in sorted(attrs.items()): - val = _get_val(v) if isinstance(v, dict) else v - rows.append(f'{escape(str(k))}: {escape(str(val))}') - content = "
".join(rows) - return f"
{len(attrs)} attributes
{content}
" - - -def render_issues_html(issues: list[str]) -> str: - parts = [HTML_HEAD.format(title="Traces")] - parts.append('
') - parts.append("

Traced Issues

") - if not issues: - parts.append("

No traces recorded yet.

") - else: - parts.append("") - parts.append(HTML_FOOT) - return "".join(parts) - - -def render_spans_html(issue: str, spans: list[dict], params: dict) -> str: - parts = [HTML_HEAD.format(title=f"Traces — {escape(issue)}")] - parts.append('
← All issues
') - parts.append(f"

{escape(issue)}

") - - filters = [f"{k}={escape(v)}" for k in ("agent_type", "trace_id", "name", "last") if (v := params.get(k))] - if filters: - parts.append(f"

Filters: {', '.join(filters)}

") - - parts.append(f"

{len(spans)} span(s)

") - - if spans: - parts.append('
') - parts.append( - "" - ) - parts.append('') - parts.append("
") - parts.append('') - parts.append("") - parts.append('') - parts.append('') - parts.append("") - parts.append("") - for s in spans: - sc = s.get("status_code", 0) - detail = _extract_detail(s.get("attributes", {}), s["name"]) - tid = s["trace_id"] - parts.append("") - parts.append(f"") - parts.append(f'') - parts.append(f'') - parts.append(f'') - parts.append(f'') - parts.append(f'') - parts.append( - f'" - ) - parts.append(f"") - parts.append("") - parts.append("
DetailStatusStartDurationAgentNameTrace IDAttributes
{detail or ''}{STATUS_LABELS.get(sc, sc)}{_fmt_time(s["start_time"])}{_fmt_duration(s["start_time"], s.get("end_time"))}{escape(s.get("agent_type") or "-")}{escape(s["name"])}' - f"{escape(tid[:12])}…{_render_attrs(s.get('attributes', {}))}
") - - parts.append(HTML_FOOT) - return "".join(parts) diff --git a/trace_server/server.py b/trace_server/server.py index a65b86f5b..69c4b16dd 100644 --- a/trace_server/server.py +++ b/trace_server/server.py @@ -41,6 +41,8 @@ (e.g. TriageAgent,think,final_answer). last Return only the N most recent traces (by earliest span start time). + since Only return spans with start_time >= this value + (nanosecond timestamp). Useful for incremental polling. Examples: curl https://trace-server.example.com/traces/RHEL-12345 @@ -64,10 +66,9 @@ import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from urllib.parse import parse_qs, urlparse -from renderer import render_issues_html, render_spans_html - logger = logging.getLogger(__name__) DB_PATH = os.environ.get("TRACE_DB_PATH", "/data/traces.db") @@ -75,7 +76,18 @@ LOG_LEVEL = os.environ.get("TRACE_LOG_LEVEL", "INFO").upper() MAX_PAYLOAD_SIZE = 100 * 1024 * 1024 # 100 MB MAX_LAST_TRACES = 900 +STATIC_DIR = Path(__file__).resolve().parent / "static" +_MIME_TYPES = { + ".html": "text/html; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".json": "application/json", + ".svg": "image/svg+xml", + ".png": "image/png", + ".ico": "image/x-icon", +} _STATUS_CODE_NAMES = {"STATUS_CODE_UNSET": 0, "STATUS_CODE_OK": 1, "STATUS_CODE_ERROR": 2} +_SQL_VAR_LIMIT = 500 _local = threading.local() @@ -98,7 +110,7 @@ def configure_logging() -> None: def init_db() -> None: - os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True) + Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True) logger.debug("Initializing database at %s", DB_PATH) db = sqlite3.connect(DB_PATH) db.execute(""" @@ -348,6 +360,59 @@ def ingest_spans(otlp_data: dict) -> int: "INSERT OR IGNORE INTO span_issues (trace_id, span_id, jira_issue) VALUES (?, ?, ?)", issue_rows, ) + # Propagate agent_type to spans missing it in affected traces + trace_ids = list({s.trace_id for s in spans}) + agent_rows: list = [] + null_rows: list = [] + for i in range(0, len(trace_ids), _SQL_VAR_LIMIT): + chunk = trace_ids[i : i + _SQL_VAR_LIMIT] + ph = ",".join("?" * len(chunk)) + agent_rows.extend( + db.execute( + f"SELECT trace_id, span_id, agent_type FROM spans " # noqa: S608 + f"WHERE trace_id IN ({ph}) AND agent_type IS NOT NULL", + chunk, + ).fetchall() + ) + null_rows.extend( + db.execute( + f"SELECT trace_id, span_id, parent_span_id FROM spans " # noqa: S608 + f"WHERE trace_id IN ({ph}) AND agent_type IS NULL", + chunk, + ).fetchall() + ) + if agent_rows and null_rows: + children: dict[tuple[str, str], list[dict]] = {} + for row in null_rows: + if row["parent_span_id"]: + children.setdefault((row["trace_id"], row["parent_span_id"]), []).append(row) + updates = [] + for ar in agent_rows: + stack = list(children.get((ar["trace_id"], ar["span_id"]), [])) + while stack: + child = stack.pop() + updates.append((ar["agent_type"], child["trace_id"], child["span_id"])) + stack.extend(children.get((child["trace_id"], child["span_id"]), [])) + if updates: + db.executemany( + "UPDATE spans SET agent_type = ? WHERE trace_id = ? AND span_id = ?", + updates, + ) + + # Propagate jira_issues to previously-stored spans in the same traces + all_issues_by_trace: dict[str, set[str]] = {} + for s in spans: + if s.jira_issues: + all_issues_by_trace.setdefault(s.trace_id, set()).update(s.jira_issues) + if all_issues_by_trace: + for trace_id, new_issues in all_issues_by_trace.items(): + for issue in new_issues: + db.execute( + "INSERT OR IGNORE INTO span_issues (trace_id, span_id, jira_issue) " + "SELECT trace_id, span_id, ? FROM spans WHERE trace_id = ?", + (issue, trace_id), + ) + db.commit() all_issues = {issue for s in spans for issue in s.jira_issues} logger.debug( @@ -367,6 +432,47 @@ def query_issues() -> list[str]: def query_spans(issue: str, params: dict) -> list[dict]: db = get_db() + # When issue is '_', query by trace_id directly (no issue association) + if issue == "_": + trace_id = params.get("trace_id") + if not trace_id: + return [] + query_bindings: list = [trace_id] + subquery = "SELECT ? AS trace_id" + + since_filter = "" + if since_ns := params.get("since"): + try: + since_filter = " AND start_time >= ?" + query_bindings.append(int(since_ns)) + except (ValueError, TypeError): + pass + + rows = db.execute( + f"""SELECT trace_id, span_id, parent_span_id, name, start_time, + end_time, status_code, jira_issue, agent_type, attributes + FROM spans + WHERE trace_id IN ({subquery}){since_filter} + ORDER BY start_time""", # noqa: S608 + query_bindings, + ).fetchall() + + return [ + { + "trace_id": r["trace_id"], + "span_id": r["span_id"], + "parent_span_id": r["parent_span_id"], + "name": r["name"], + "start_time": r["start_time"], + "end_time": r["end_time"], + "status_code": r["status_code"], + "jira_issue": r["jira_issue"], + "agent_type": r["agent_type"], + "attributes": json.loads(r["attributes"]), + } + for r in rows + ] + # Find trace IDs via the span_issues junction table, then apply filters on spans issue_conditions = ["si.jira_issue = ?"] issue_bindings: list = [issue] @@ -420,12 +526,20 @@ def query_spans(issue: str, params: dict) -> list[dict]: subquery = f"SELECT DISTINCT trace_id FROM span_issues si WHERE {join_where}" # noqa: S608 query_bindings = bindings + since_filter = "" + if since_ns := params.get("since"): + try: + since_filter = " AND start_time >= ?" + query_bindings.append(int(since_ns)) + except (ValueError, TypeError): + pass + # Fetch ALL spans from matching traces rows = db.execute( f"""SELECT trace_id, span_id, parent_span_id, name, start_time, end_time, status_code, jira_issue, agent_type, attributes FROM spans - WHERE trace_id IN ({subquery}) + WHERE trace_id IN ({subquery}){since_filter} ORDER BY start_time""", # noqa: S608 query_bindings, ).fetchall() @@ -454,37 +568,63 @@ def query_spans(issue: str, params: dict) -> list[dict]: def query_recent_traces(since_ns: int, workflow: str | None, limit: int) -> list[dict]: - """Return recent root workflow spans, optionally filtered by workflow name.""" + """Return recent traces, including in-progress ones whose root span hasn't arrived yet.""" db = get_db() - conditions = ["parent_span_id = ''", "name LIKE '%Workflow'", "start_time >= ?"] - bindings: list = [since_ns] + effective_limit = min(limit, MAX_LAST_TRACES) + # Completed traces: root workflow span exists + root_conditions = ["parent_span_id = ''", "name LIKE '%Workflow'", "start_time >= ?"] + root_bindings: list = [since_ns] if workflow: - conditions.append("name = ?") - bindings.append(workflow) - - where = " AND ".join(conditions) - bindings.append(min(limit, MAX_LAST_TRACES)) + root_conditions.append("name = ?") + root_bindings.append(workflow) + root_where = " AND ".join(root_conditions) - rows = db.execute( + root_rows = db.execute( f"""SELECT s.trace_id, s.name, s.start_time, s.end_time, s.status_code FROM spans s - WHERE {where} + WHERE {root_where} ORDER BY s.start_time DESC LIMIT ?""", # noqa: S608 - bindings, + [*root_bindings, effective_limit], ).fetchall() - if not rows: + root_trace_ids = {r["trace_id"] for r in root_rows} + + # In-progress traces: have spans linked to jira issues in the time window + # but no root Workflow span yet + inprog_filter = "" + inprog_bindings: list = [since_ns] + if workflow: + wf_base = workflow.removesuffix("Workflow").lower() + inprog_filter = " AND json_extract(s.attributes, '$.\"workflow.name\".stringValue') = ?" + inprog_bindings.append(wf_base) + inprog_rows = db.execute( + f"""SELECT s.trace_id, MIN(s.start_time) as first_start, + MAX(json_extract(s.attributes, '$."workflow.name".stringValue')) as workflow_name + FROM spans s + JOIN span_issues si ON s.trace_id = si.trace_id AND s.span_id = si.span_id + WHERE s.start_time >= ?{inprog_filter} + GROUP BY s.trace_id + HAVING s.trace_id NOT IN ( + SELECT trace_id FROM spans + WHERE parent_span_id = '' AND name LIKE '%Workflow' + ) + ORDER BY first_start DESC + LIMIT ?""", # noqa: S608 + [*inprog_bindings, effective_limit], + ).fetchall() + + all_trace_ids = list(root_trace_ids | {r["trace_id"] for r in inprog_rows}) + if not all_trace_ids: return [] - trace_ids = [r["trace_id"] for r in rows] - ph = ",".join("?" * len(trace_ids)) + ph = ",".join("?" * len(all_trace_ids)) issue_rows = db.execute( f"SELECT trace_id, jira_issue FROM span_issues WHERE trace_id IN ({ph}) " # noqa: S608 "GROUP BY trace_id, jira_issue", - trace_ids, + all_trace_ids, ).fetchall() issues_by_trace: dict[str, list[str]] = {} for ir in issue_rows: @@ -494,12 +634,12 @@ def query_recent_traces(since_ns: int, workflow: str | None, limit: int) -> list f"SELECT trace_id, COUNT(*) as cnt, " # noqa: S608 "SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) as errors " f"FROM spans WHERE trace_id IN ({ph}) GROUP BY trace_id", - trace_ids, + all_trace_ids, ).fetchall() counts_by_trace = {cr["trace_id"]: cr for cr in count_rows} results = [] - for r in rows: + for r in root_rows: tid = r["trace_id"] counts = counts_by_trace.get(tid) results.append( @@ -515,7 +655,29 @@ def query_recent_traces(since_ns: int, workflow: str | None, limit: int) -> list } ) - return results + for r in inprog_rows: + tid = r["trace_id"] + if tid in root_trace_ids: + continue + counts = counts_by_trace.get(tid) + wf_name = r["workflow_name"] + if wf_name: + wf_name = wf_name[0].upper() + wf_name[1:] + "Workflow" + results.append( + { + "trace_id": tid, + "workflow": wf_name or "(in progress)", + "issues": sorted(issues_by_trace.get(tid, [])), + "start_time": r["first_start"], + "end_time": None, + "status_code": 0, + "num_spans": counts["cnt"] if counts else 0, + "error_count": counts["errors"] if counts else 0, + } + ) + + results.sort(key=lambda x: x["start_time"] or 0, reverse=True) + return results[:effective_limit] class TraceHandler(BaseHTTPRequestHandler): @@ -580,10 +742,6 @@ def do_POST(self): logger.debug("POST %s not found", self.path) self._send_json(404, {"error": "not found"}) - def _wants_html(self) -> bool: - accept = self.headers.get("Accept", "") - return "text/html" in accept - def do_GET(self): parsed = urlparse(self.path) path = parsed.path.rstrip("/") @@ -591,7 +749,16 @@ def do_GET(self): logger.debug("GET %s params=%r", self.path, params) - if path == "/health": + if path == "" or path == "/index.html": + self._send_file(STATIC_DIR / "index.html") + elif path.startswith("/static/"): + rel = path[len("/static/") :] + filepath = (STATIC_DIR / rel).resolve() + if not filepath.is_relative_to(STATIC_DIR.resolve()): + self._send_json(404, {"error": "not found"}) + return + self._send_file(filepath) + elif path == "/health": self._send_json(200, {"status": "ok"}) elif path == "/traces/recent": try: @@ -612,7 +779,7 @@ def do_GET(self): return logger.debug("recent_traces returned %d traces", len(traces)) self._send_json(200, {"traces": traces, "count": len(traces)}) - elif path == "/traces" or path == "": + elif path == "/traces": try: issues = query_issues() except Exception as e: @@ -620,10 +787,7 @@ def do_GET(self): self._send_json(500, {"error": f"failed to query issues: {e}"}) return logger.debug("Listed %d issues", len(issues)) - if self._wants_html(): - self._send_html(200, render_issues_html(issues)) - else: - self._send_json(200, {"issues": issues}) + self._send_json(200, {"issues": issues}) elif path.startswith("/traces/"): issue = path[len("/traces/") :] try: @@ -632,10 +796,7 @@ def do_GET(self): logger.exception("Failed to query spans for %s", issue) self._send_json(500, {"error": f"failed to query spans: {e}"}) return - if self._wants_html(): - self._send_html(200, render_spans_html(issue, spans, params)) - else: - self._send_json(200, {"spans": spans, "count": len(spans)}) + self._send_json(200, {"spans": spans, "count": len(spans)}) else: logger.debug("GET %s not found", self.path) self._send_json(404, {"error": "not found"}) @@ -648,10 +809,19 @@ def _send_json(self, code: int, data: dict): self.end_headers() self.wfile.write(body) - def _send_html(self, code: int, html: str): - body = html.encode() - self.send_response(code) - self.send_header("Content-Type", "text/html; charset=utf-8") + def _send_file(self, filepath: Path): + if not filepath.is_file(): + self._send_json(404, {"error": "not found"}) + return + try: + body = filepath.read_bytes() + except OSError: + self._send_json(404, {"error": "not found"}) + return + ext = filepath.suffix.lower() + content_type = _MIME_TYPES.get(ext, "application/octet-stream") + self.send_response(200) + self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) diff --git a/trace_server/static/app.js b/trace_server/static/app.js new file mode 100644 index 000000000..94d521db4 --- /dev/null +++ b/trace_server/static/app.js @@ -0,0 +1,1385 @@ +'use strict'; + +// ============================================================ +// Utilities +// ============================================================ + +function getVal(value) { + if (!value || typeof value !== 'object') return null; + for (const k of ['stringValue', 'intValue', 'boolValue', 'doubleValue']) { + if (k in value) return value[k]; + } + if ('arrayValue' in value && value.arrayValue && value.arrayValue.values) { + return value.arrayValue.values.map(getVal); + } + return null; +} + +function escHtml(s) { + if (typeof s !== 'string') s = String(s ?? ''); + return s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); +} + +function lazyDetails(summaryText, buildFn, open) { + const d = document.createElement('details'); + const s = document.createElement('summary'); + s.textContent = summaryText; + d.appendChild(s); + let populated = open; + d.addEventListener('toggle', () => { + if (d.open) { + if (!populated) d.appendChild(buildFn()); + populated = true; + } else { + while (d.lastChild !== s) d.removeChild(d.lastChild); + populated = false; + } + }); + if (open) { + d.appendChild(buildFn()); + d.open = true; + } + return d; +} + +function el(tag, attrs) { + const e = document.createElement(tag); + if (attrs) { + for (const [k, v] of Object.entries(attrs)) { + if (k === 'className') e.className = v; + else if (k === 'textContent') e.textContent = v; + else if (k === 'innerHTML') e.innerHTML = v; + else if (k.startsWith('on')) e.addEventListener(k.slice(2).toLowerCase(), v); + else e.setAttribute(k, v); + } + } + for (let i = 2; i < arguments.length; i++) { + const child = arguments[i]; + if (child == null) continue; + if (typeof child === 'string') e.appendChild(document.createTextNode(child)); + else if (child instanceof Node) e.appendChild(child); + } + return e; +} + +function fmtTime(nanos) { + if (!nanos) return '-'; + const d = new Date(nanos / 1e6); + const pad = n => String(n).padStart(2, '0'); + return d.getFullYear() + '-' + pad(d.getMonth() + 1) + '-' + pad(d.getDate()) + ' ' + + pad(d.getHours()) + ':' + pad(d.getMinutes()) + ':' + pad(d.getSeconds()); +} + +function fmtTimeShort(nanos) { + if (!nanos) return '-'; + const d = new Date(nanos / 1e6); + const pad = n => String(n).padStart(2, '0'); + return pad(d.getHours()) + ':' + pad(d.getMinutes()) + ':' + pad(d.getSeconds()); +} + +function fmtDuration(startNanos, endNanos) { + if (!endNanos || !startNanos) return '-'; + const ms = (endNanos - startNanos) / 1e6; + if (ms < 1000) return ms.toFixed(0) + 'ms'; + if (ms < 60000) return (ms / 1000).toFixed(1) + 's'; + return (ms / 60000).toFixed(1) + 'm'; +} + +function fmtTokens(n) { + if (n >= 1e6) return (n / 1e6).toFixed(1) + 'M'; + if (n >= 1e3) return (n / 1e3).toFixed(1) + 'k'; + return String(n); +} + +function aggregateLlmStats(spans) { + let calls = 0, promptTokens = 0, completionTokens = 0, cost = 0, cacheRead = 0, cacheWrite = 0; + const models = new Set(); + for (const s of spans) { + const attrs = s.attributes || {}; + const kind = getVal(attrs['openinference.span.kind']); + if (kind !== 'LLM' && !s.name.endsWith('ChatModel')) continue; + calls++; + const pt = getVal(attrs['llm.token_count.prompt']); + const ct = getVal(attrs['llm.token_count.completion']); + const tc = getVal(attrs['llm.cost.total']); + const cr = getVal(attrs['metadata.usage.cached_prompt_tokens']); + const cw = getVal(attrs['metadata.usage.cached_creation_tokens']); + const model = getVal(attrs['llm.model_name']); + if (pt) promptTokens += Number(pt); + if (ct) completionTokens += Number(ct); + if (tc) cost += Number(tc); + if (cr) cacheRead += Number(cr); + if (cw) cacheWrite += Number(cw); + if (model) models.add(model); + } + return {calls, promptTokens, completionTokens, cost, cacheRead, cacheWrite, models}; +} + +function isTraceComplete(spans) { + return spans.some(s => (!s.parent_span_id || s.parent_span_id === '') && s.end_time); +} + +function fmtAgo(nanos) { + if (!nanos) return ''; + const ms = Date.now() - nanos / 1e6; + if (ms < 60000) return 'just now'; + if (ms < 3600000) return Math.floor(ms / 60000) + 'm ago'; + if (ms < 86400000) return Math.floor(ms / 3600000) + 'h ago'; + return Math.floor(ms / 86400000) + 'd ago'; +} + +function fmtToolCall(name, inputValue) { + try { + const parsed = JSON.parse(inputValue); + const args = parsed.input || parsed; + const parts = []; + for (const [k, v] of Object.entries(args)) { + let vs; + if (v === null) vs = 'null'; + else if (typeof v === 'string') { + vs = '"' + (v.length > 60 ? v.slice(0, 57) + '…' : v) + '"'; + } else vs = JSON.stringify(v); + parts.push(k + '=' + vs); + } + const argsStr = parts.join(', '); + const full = name + '(' + argsStr + ')'; + if (full.length > 120) return {name, args: argsStr.slice(0, 120 - name.length - 2) + '…'}; + return {name, args: argsStr}; + } catch (e) { + return {name, args: null}; + } +} + +function statusClass(code) { + if (code === 1) return 'ok'; + if (code === 2) return 'error'; + return ''; +} + +function statusLabel(code) { + if (code === 1) return 'Ok'; + if (code === 2) return 'Error'; + return 'Unset'; +} + +// ============================================================ +// API Client +// ============================================================ + +const api = { + async recentTraces(opts) { + const p = new URLSearchParams(); + if (opts.since) p.set('since', opts.since); + if (opts.workflow) p.set('workflow', opts.workflow); + if (opts.limit) p.set('limit', opts.limit); + const qs = p.toString(); + const res = await fetch('/traces/recent' + (qs ? '?' + qs : '')); + if (!res.ok) throw new Error(await res.text()); + return res.json(); + }, + + async issues() { + const res = await fetch('/traces/'); + if (!res.ok) throw new Error(await res.text()); + return res.json(); + }, + + async spans(issue, opts) { + const p = new URLSearchParams(); + if (opts.traceId) p.set('trace_id', opts.traceId); + if (opts.agentType) p.set('agent_type', opts.agentType); + if (opts.name) p.set('name', opts.name); + if (opts.last) p.set('last', opts.last); + if (opts.since) p.set('since', opts.since); + const qs = p.toString(); + const res = await fetch('/traces/' + encodeURIComponent(issue) + (qs ? '?' + qs : '')); + if (!res.ok) throw new Error(await res.text()); + return res.json(); + }, +}; + +// ============================================================ +// State +// ============================================================ + +const state = { + view: 'recent', + recentTraces: [], + currentTraceId: null, + currentIssue: null, + spans: [], + spanIds: new Set(), + pollTimeoutId: null, + pollGen: 0, + filterSince: 10800, + filterWorkflow: '', + issueFilterSince: 86400, + previousHash: '#/', +}; + +// ============================================================ +// Polling +// ============================================================ + +function stopPolling() { + state.pollGen++; + if (state.pollTimeoutId) { + clearTimeout(state.pollTimeoutId); + state.pollTimeoutId = null; + } +} + +function startPolling(fn, intervalMs) { + stopPolling(); + const gen = state.pollGen; + async function tick() { + if (state.pollGen !== gen) return; + try { await fn(); } catch (e) {} + if (state.pollGen !== gen) return; + state.pollTimeoutId = setTimeout(tick, intervalMs); + } + state.pollTimeoutId = setTimeout(tick, intervalMs); +} + +document.addEventListener('visibilitychange', () => { + if (document.hidden) { + stopPolling(); + } else if (state.view === 'trace' && state.currentIssue && state.currentTraceId) { + if (!isTraceComplete(state.spans)) { + startPolling(() => pollNewSpans(state.currentIssue, state.currentTraceId), 5000); + } + } else if (state.view === 'recent' || state.view === 'issues' || state.view === 'issue') { + route(); + } +}); + +// ============================================================ +// Router +// ============================================================ + +function route() { + stopPolling(); + window.removeEventListener('scroll', onScroll); + if (sidebarScrollHandler) { window.removeEventListener('scroll', sidebarScrollHandler); sidebarScrollHandler = null; } + const hash = location.hash || '#/'; + const app = document.getElementById('app'); + app.innerHTML = ''; + + document.querySelector('.jump-bottom')?.remove(); + + const prevHash = state.previousHash; + state.previousHash = hash; + + if (hash.startsWith('#/trace/')) { + const parts = hash.slice(8).split('/'); + const traceId = parts.pop(); + const issue = decodeURIComponent(parts.join('/')); + state.view = 'trace'; + state.currentIssue = issue; + state.currentTraceId = traceId; + renderTraceDetail(app, issue, traceId, prevHash); + } else if (hash.startsWith('#/issues/')) { + const issue = decodeURIComponent(hash.slice(9)); + state.view = 'issue'; + state.currentIssue = issue; + renderIssueDetail(app, issue); + } else if (hash === '#/issues') { + state.view = 'issues'; + renderIssues(app); + } else { + state.view = 'recent'; + renderRecent(app); + } + + updateNav(); +} + +function updateNav() { + document.querySelectorAll('.header-nav a').forEach(a => { + const href = a.getAttribute('href'); + if (state.view === 'recent' && href === '#/') a.classList.add('active'); + else if ((state.view === 'issues' || state.view === 'issue') && href === '#/issues') a.classList.add('active'); + else a.classList.remove('active'); + }); +} + +window.addEventListener('hashchange', route); + +// ============================================================ +// Header +// ============================================================ + +function renderHeader() { + const header = document.getElementById('header'); + header.innerHTML = ''; + const logo = el('a', {className: 'header-logo', href: '#/'}); + logo.appendChild(el('img', {className: 'header-avatar', src: '/static/ymir-avatar.png'})); + logo.appendChild(document.createTextNode('Ymir traces')); + header.appendChild(logo); + header.appendChild(el('nav', {className: 'header-nav'}, + el('a', {href: '#/'}, 'recent'), + el('a', {href: '#/issues'}, 'issues'))); + header.appendChild(el('div', {className: 'header-spacer'})); + header.appendChild(el('span', {className: 'header-status', id: 'header-status'})); + header.appendChild(el('button', { + className: 'header-btn', + onClick: toggleTheme, + id: 'theme-btn', + }, themeLabel())); +} + +function themeLabel() { + return document.documentElement.classList.contains('dark') ? '[light]' : '[dark]'; +} + +function toggleTheme() { + document.documentElement.classList.toggle('dark'); + const isDark = document.documentElement.classList.contains('dark'); + try { localStorage.setItem('theme', isDark ? 'dark' : 'light'); } catch(e) {} + const btn = document.getElementById('theme-btn'); + if (btn) btn.textContent = themeLabel(); +} + +function setStatus(text) { + const el = document.getElementById('header-status'); + if (el) el.textContent = text; +} + +// ============================================================ +// Recent Traces View +// ============================================================ + +async function renderRecent(container) { + container.appendChild(el('div', {className: 'loading'}, 'loading traces...')); + + const filtersBar = el('div', {className: 'filters'}); + + const sinceSelect = el('select', {className: 'filter-select', onChange: (e) => { + state.filterSince = parseInt(e.target.value); + refreshRecent(container); + }}); + for (const [label, val] of [['1h', 3600], ['3h', 10800], ['12h', 43200], ['24h', 86400], ['3d', 259200], ['7d', 604800], ['30d', 2592000]]) { + const opt = el('option', {value: val}, label); + if (val === state.filterSince) opt.selected = true; + sinceSelect.appendChild(opt); + } + + const workflowInput = el('input', { + className: 'filter-input', + type: 'text', + placeholder: 'workflow filter', + value: state.filterWorkflow, + onInput: (e) => { + state.filterWorkflow = e.target.value; + }, + onKeydown: (e) => { + if (e.key === 'Enter') refreshRecent(container); + }, + }); + + filtersBar.appendChild(el('span', {className: 'filter-label'}, 'since:')); + filtersBar.appendChild(sinceSelect); + filtersBar.appendChild(el('span', {className: 'filter-label'}, 'workflow:')); + filtersBar.appendChild(workflowInput); + + try { + const data = await api.recentTraces({ + since: state.filterSince, + workflow: state.filterWorkflow || undefined, + }); + state.recentTraces = data.traces || []; + container.innerHTML = ''; + container.appendChild(filtersBar); + renderTraceCards(container, state.recentTraces); + setStatus(state.recentTraces.length + ' traces'); + } catch (e) { + container.innerHTML = ''; + container.appendChild(filtersBar); + container.appendChild(el('div', {className: 'error-banner'}, 'Failed to load traces: ' + e.message)); + } + + startPolling(() => refreshRecent(container), 30000); +} + +async function refreshRecent(container) { + try { + const data = await api.recentTraces({ + since: state.filterSince, + workflow: state.filterWorkflow || undefined, + }); + state.recentTraces = data.traces || []; + const grid = container.querySelector('.trace-grid'); + const filters = container.querySelector('.filters'); + container.innerHTML = ''; + if (filters) container.appendChild(filters); + renderTraceCards(container, state.recentTraces); + setStatus(state.recentTraces.length + ' traces'); + } catch (e) { + // silently skip failed polls + } +} + +function renderTraceCards(container, traces) { + if (traces.length === 0) { + container.appendChild(el('div', {className: 'empty-state'}, 'No traces found.')); + return; + } + + const grid = el('div', {className: 'trace-grid'}); + for (const t of traces) { + const sc = statusClass(t.status_code); + const card = el('div', { + className: 'trace-card' + (sc ? ' status-' + sc : ''), + onClick: () => { + const issue = (t.issues && t.issues[0]) || '_'; + location.hash = '#/trace/' + encodeURIComponent(issue) + '/' + t.trace_id; + }, + }); + + card.appendChild(el('div', {className: 'trace-card-header'}, + el('span', {className: 'status-dot ' + sc}), + el('span', {className: 'trace-card-workflow'}, t.workflow || 'unknown'), + el('span', {className: 'trace-card-time'}, fmtAgo(t.start_time)), + )); + + if (t.issues && t.issues.length > 0) { + const badges = el('div', {className: 'trace-card-issues'}); + for (const issue of t.issues) { + badges.appendChild(el('a', { + className: 'issue-badge', + href: '#/issues/' + encodeURIComponent(issue), + onClick: (e) => e.stopPropagation(), + }, issue)); + } + card.appendChild(badges); + } + + const meta = el('div', {className: 'trace-card-meta'}); + meta.appendChild(el('span', {}, fmtDuration(t.start_time, t.end_time))); + meta.appendChild(el('span', {}, t.num_spans + ' spans')); + if (t.error_count > 0) { + meta.appendChild(el('span', {className: 'error-count'}, t.error_count + ' errors')); + } + card.appendChild(meta); + + grid.appendChild(card); + } + container.appendChild(grid); +} + +// ============================================================ +// Trace Detail View +// ============================================================ + +async function renderTraceDetail(container, issue, traceId, prevHash) { + container.appendChild(el('div', {className: 'loading'}, 'loading spans...')); + + try { + const data = await api.spans(issue, {traceId: traceId}); + state.spans = data.spans || []; + state.spanIds = new Set(state.spans.map(s => s.span_id)); + + container.innerHTML = ''; + + const tree = buildSpanTree(state.spans); + const agents = collectAgents(tree, 0); + + const hasSidebar = agents.length > 0; + const layout = el('div', {className: 'trace-layout'}); + if (!hasSidebar) layout.style.gridTemplateColumns = '1fr'; + if (hasSidebar) layout.appendChild(renderSidebar(agents)); + const main = el('div', {className: 'trace-main'}); + + let backHref = '#/'; + let backLabel = '← recent traces'; + if (prevHash && prevHash.startsWith('#/issues/')) { + backHref = prevHash; + backLabel = '← ' + decodeURIComponent(prevHash.slice(9)); + } + main.appendChild(el('a', {className: 'back-link', href: backHref}, backLabel)); + const header = el('div', {className: 'trace-detail-header'}); + header.appendChild(el('h1', {}, issue + ' / ' + traceId)); + const meta = el('div', {className: 'trace-detail-meta'}); + if (state.spans.length > 0) { + const first = state.spans[0]; + meta.appendChild(el('span', {}, 'started: ' + fmtTime(first.start_time))); + meta.appendChild(el('span', {}, 'spans: ' + state.spans.length)); + const errors = state.spans.filter(s => s.status_code === 2).length; + if (errors > 0) meta.appendChild(el('span', {className: 'error-count'}, 'errors: ' + errors)); + const llmStats = aggregateLlmStats(state.spans); + if (llmStats.calls > 0) { + meta.appendChild(el('span', {}, 'LLM calls: ' + llmStats.calls)); + let tokenSummary = 'tokens: ' + fmtTokens(llmStats.promptTokens) + ' in'; + if (llmStats.cacheRead || llmStats.cacheWrite) { + const parts = []; + if (llmStats.cacheRead) parts.push(fmtTokens(llmStats.cacheRead) + ' cached'); + if (llmStats.cacheWrite) parts.push(fmtTokens(llmStats.cacheWrite) + ' new'); + tokenSummary += ' (' + parts.join(', ') + ')'; + } + tokenSummary += ' / ' + fmtTokens(llmStats.completionTokens) + ' out'; + meta.appendChild(el('span', {}, tokenSummary)); + if (llmStats.cost > 0) meta.appendChild(el('span', {}, 'cost: $' + llmStats.cost.toFixed(2))); + if (llmStats.models.size > 0) meta.appendChild(el('span', {}, 'model: ' + [...llmStats.models].join(', '))); + } + } + header.appendChild(meta); + main.appendChild(header); + + const spanList = el('div', {className: 'span-list', id: 'span-list'}); + renderSpanTree(spanList, tree, 0); + main.appendChild(spanList); + layout.appendChild(main); + container.appendChild(layout); + + setupAutoScroll(); + if (isTraceComplete(state.spans)) { + setStatus('completed · ' + state.spans.length + ' spans'); + } else { + startPolling(() => pollNewSpans(issue, traceId), 5000); + setStatus('live'); + } + } catch (e) { + container.innerHTML = ''; + container.appendChild(el('div', {className: 'error-banner'}, 'Failed to load spans: ' + e.message)); + } +} + +async function pollNewSpans(issue, traceId) { + try { + const data = await api.spans(issue, {traceId: traceId}); + const allSpans = data.spans || []; + let changed = false; + + const existingById = new Map(); + for (const s of state.spans) existingById.set(s.span_id, s); + + for (const s of allSpans) { + const existing = existingById.get(s.span_id); + if (!existing) { + state.spans.push(s); + state.spanIds.add(s.span_id); + changed = true; + } else if (existing.end_time !== s.end_time || existing.status_code !== s.status_code) { + Object.assign(existing, s); + changed = true; + } + } + if (!changed) return; + state.spans.sort((a, b) => (a.start_time || 0) - (b.start_time || 0)); + + // Re-render full tree so placeholders and hierarchy stay correct + const spanList = document.getElementById('span-list'); + if (!spanList) return; + const scrollY = window.scrollY; + spanList.innerHTML = ''; + const tree = buildSpanTree(state.spans); + renderSpanTree(spanList, tree, 0); + window.scrollTo({top: scrollY}); + + const agents = collectAgents(tree, 0); + const oldSidebar = document.getElementById('trace-sidebar'); + if (agents.length > 0 && oldSidebar) { + oldSidebar.replaceWith(renderSidebar(agents)); + } else if (agents.length > 0 && !oldSidebar) { + const layout = spanList.closest('.trace-layout'); + if (layout) layout.insertBefore(renderSidebar(agents), layout.firstChild); + } + + const meta = document.querySelector('.trace-detail-meta'); + if (meta) { + meta.innerHTML = ''; + const first = state.spans[0]; + if (first) meta.appendChild(el('span', {}, 'started: ' + fmtTime(first.start_time))); + meta.appendChild(el('span', {}, 'spans: ' + state.spans.length)); + const errors = state.spans.filter(s => s.status_code === 2).length; + if (errors > 0) meta.appendChild(el('span', {className: 'error-count'}, 'errors: ' + errors)); + const llmStats = aggregateLlmStats(state.spans); + if (llmStats.calls > 0) { + meta.appendChild(el('span', {}, 'LLM calls: ' + llmStats.calls)); + let tokenSummary = 'tokens: ' + fmtTokens(llmStats.promptTokens) + ' in'; + if (llmStats.cacheRead || llmStats.cacheWrite) { + const parts = []; + if (llmStats.cacheRead) parts.push(fmtTokens(llmStats.cacheRead) + ' cached'); + if (llmStats.cacheWrite) parts.push(fmtTokens(llmStats.cacheWrite) + ' new'); + tokenSummary += ' (' + parts.join(', ') + ')'; + } + tokenSummary += ' / ' + fmtTokens(llmStats.completionTokens) + ' out'; + meta.appendChild(el('span', {}, tokenSummary)); + if (llmStats.cost > 0) meta.appendChild(el('span', {}, 'cost: $' + llmStats.cost.toFixed(2))); + if (llmStats.models.size > 0) meta.appendChild(el('span', {}, 'model: ' + [...llmStats.models].join(', '))); + } + } + + maybeAutoScroll(); + if (isTraceComplete(state.spans)) { + stopPolling(); + setStatus('completed · ' + state.spans.length + ' spans'); + } else { + setStatus('live · ' + state.spans.length + ' spans'); + } + } catch (e) { + // silently skip failed polls + } +} + +// ============================================================ +// Span Tree +// ============================================================ + +function buildSpanTree(spans) { + const byId = new Map(); + const roots = []; + for (const span of spans) { + byId.set(span.span_id, {...span, children: []}); + } + // Synthesize placeholders for missing parents + const missingParents = new Map(); + for (const span of spans) { + const pid = span.parent_span_id; + if (pid && !byId.has(pid)) { + if (!missingParents.has(pid)) missingParents.set(pid, []); + missingParents.get(pid).push(span); + } + } + const placeholders = []; + for (const [pid, children] of missingParents) { + const earliest = Math.min(...children.map(c => c.start_time)); + const name = children.map(c => getVal((c.attributes || {})['agent.name'])).find(Boolean) || null; + const placeholder = { + trace_id: children[0].trace_id, span_id: pid, parent_span_id: '', + name, start_time: earliest, end_time: null, status_code: 0, + jira_issue: null, agent_type: null, attributes: {}, _placeholder: true, + }; + placeholders.push(placeholder); + byId.set(pid, {...placeholder, children: []}); + } + // Nest placeholders in the same trace: earliest is the root + const byTrace = new Map(); + for (const p of placeholders) { + if (!byTrace.has(p.trace_id)) byTrace.set(p.trace_id, []); + byTrace.get(p.trace_id).push(p); + } + for (const group of byTrace.values()) { + if (group.length > 1) { + group.sort((a, b) => a.start_time - b.start_time); + const rootPid = group[0].span_id; + for (let i = 1; i < group.length; i++) { + group[i].parent_span_id = rootPid; + byId.get(group[i].span_id).parent_span_id = rootPid; + } + } + } + // Name placeholders: root always from workflow.name, non-root from agent.name only + for (const p of placeholders) { + const node = byId.get(p.span_id); + const isRoot = !p.parent_span_id; + if (isRoot) { + const wfAttr = (missingParents.get(p.span_id) || []) + .map(c => getVal((c.attributes || {})['workflow.name'])).find(Boolean); + if (wfAttr) { + node.name = wfAttr[0].toUpperCase() + wfAttr.slice(1) + 'Workflow'; + } else if (!node.name) { + node.name = '(in progress)'; + } + } else if (!node.name) { + node.name = '(agent)'; + } + } + for (const p of placeholders) { + const node = byId.get(p.span_id); + const parent = byId.get(node.parent_span_id); + if (parent) { + parent.children.push(node); + } else { + roots.push(node); + } + } + // Build tree for real spans + for (const span of spans) { + const node = byId.get(span.span_id); + const parent = byId.get(span.parent_span_id); + if (parent) { + parent.children.push(node); + } else { + roots.push(node); + } + } + // Sort children by start_time + for (const node of byId.values()) { + node.children.sort((a, b) => a.start_time - b.start_time); + } + roots.sort((a, b) => a.start_time - b.start_time); + return roots; +} + +function treeLlmStats(node) { + let promptTokens = 0, completionTokens = 0, cost = 0, cacheRead = 0, cacheWrite = 0; + const attrs = node.attributes || {}; + const kind = getVal(attrs['openinference.span.kind']); + if (kind === 'LLM' || node.name.endsWith('ChatModel')) { + const pt = getVal(attrs['llm.token_count.prompt']); + const ct = getVal(attrs['llm.token_count.completion']); + const tc = getVal(attrs['llm.cost.total']); + const cr = getVal(attrs['metadata.usage.cached_prompt_tokens']); + const cw = getVal(attrs['metadata.usage.cached_creation_tokens']); + if (pt) promptTokens += Number(pt); + if (ct) completionTokens += Number(ct); + if (tc) cost += Number(tc); + if (cr) cacheRead += Number(cr); + if (cw) cacheWrite += Number(cw); + } + if (node.children) { + for (const child of node.children) { + const s = treeLlmStats(child); + promptTokens += s.promptTokens; + completionTokens += s.completionTokens; + cost += s.cost; + cacheRead += s.cacheRead; + cacheWrite += s.cacheWrite; + } + } + return {promptTokens, completionTokens, cost, cacheRead, cacheWrite}; +} + +function isEmptyLlm(node) { + const attrs = node.attributes || {}; + const kind = getVal(attrs['openinference.span.kind']); + if (kind !== 'LLM' && !node.name.endsWith('ChatModel')) return false; + if (getVal(attrs['llm.output_messages.0.message.contents.0.message_content.type'])) return false; + if (getVal(attrs['llm.output_messages.0.message.tool_calls.0.tool_call.function.name'])) return false; + return true; +} + +function renderSpanTree(container, nodes, depth, parent) { + const parentKind = parent ? getVal((parent.attributes || {})['openinference.span.kind']) : null; + for (const node of nodes) { + if (parentKind === 'TOOL' && node.name === 'error') continue; + if (isEmptyLlm(node)) continue; + container.appendChild(renderSpanRow(node, depth, parent)); + if (node.children && node.children.length > 0) { + renderSpanTree(container, node.children, depth + 1, node); + } + } +} + +function renderSpanRow(span, depth, parent) { + const attrs = span.attributes || {}; + let kind = getVal(attrs['openinference.span.kind']) || ''; + if (!kind && span.name.endsWith('ChatModel')) kind = 'LLM'; + let effectiveStatus = span.status_code; + if (kind === 'TOOL' && span.name === 'run_shell_command') { + try { + const out = JSON.parse(getVal(attrs['output.value']) || '{}'); + if (out.exit_code !== 0) effectiveStatus = 2; + } catch (e) {} + } + const sc = statusClass(effectiveStatus); + + let kindClass = ''; + if (kind === 'LLM') kindClass = 'kind-llm'; + else if (kind === 'TOOL') kindClass = 'kind-tool'; + else if (kind === 'AGENT' || kind === 'CHAIN') kindClass = 'kind-agent'; + if (effectiveStatus === 2) kindClass = 'has-error'; + + const row = el('div', { + className: 'span-row ' + kindClass, + id: 'span-' + span.span_id, + style: 'padding-left: ' + (10 + depth * 20) + 'px', + }); + + const header = el('div', {className: 'span-row-header'}); + if (kind === 'TOOL' && span.name === 'final_answer') { + header.appendChild(el('span', {className: 'span-name'}, span.name)); + } else if (kind === 'TOOL' && span.name === 'run_shell_command') { + const inputVal = getVal(attrs['input.value']); + let cmd = span.name; + try { + const parsed = JSON.parse(inputVal); + cmd = (parsed.input || parsed).command || cmd; + } catch (e) {} + const nameSpan = el('span', {className: 'span-name'}, '$ '); + nameSpan.appendChild(el('span', {className: 'span-args'}, cmd)); + header.appendChild(nameSpan); + } else if (kind === 'TOOL') { + const inputVal = getVal(attrs['input.value']); + const tc = fmtToolCall(span.name, inputVal); + const nameSpan = el('span', {className: 'span-name'}, tc.name); + if (tc.args !== null) { + nameSpan.appendChild(el('span', {className: 'span-args'}, '(' + tc.args + ')')); + } + header.appendChild(nameSpan); + } else { + header.appendChild(el('span', {className: 'span-name'}, span.name)); + } + if (kind) header.appendChild(el('span', {className: 'span-kind'}, kind)); + if (sc) header.appendChild(el('span', {className: 'span-status ' + sc}, statusLabel(effectiveStatus))); + + const isLlm = kind === 'LLM' || (!kind && span.name.endsWith('ChatModel')); + const isAgent = kind === 'AGENT' || kind === 'CHAIN' || span._placeholder; + if (isLlm) { + const pt = Number(getVal(attrs['llm.token_count.prompt']) || 0); + const ct = Number(getVal(attrs['llm.token_count.completion']) || 0); + const tc = Number(getVal(attrs['llm.cost.total']) || 0); + const cr = Number(getVal(attrs['metadata.usage.cached_prompt_tokens']) || 0); + const cw = Number(getVal(attrs['metadata.usage.cached_creation_tokens']) || 0); + if (pt || ct) { + let tokenText = fmtTokens(pt); + if (cr || cw) { + const parts = []; + if (cr) parts.push(fmtTokens(cr) + ' cached'); + if (cw) parts.push(fmtTokens(cw) + ' new'); + tokenText += ' (' + parts.join(', ') + ')'; + } + tokenText += ' → ' + fmtTokens(ct); + header.appendChild(el('span', {className: 'span-tokens'}, tokenText)); + } + if (tc > 0) { + header.appendChild(el('span', {className: 'span-cost'}, '$' + tc.toFixed(4))); + } + } else if (isAgent && span.children && span.children.length > 0) { + const stats = treeLlmStats(span); + if (stats.promptTokens || stats.completionTokens) { + let tokenText = fmtTokens(stats.promptTokens); + if (stats.cacheRead || stats.cacheWrite) { + const parts = []; + if (stats.cacheRead) parts.push(fmtTokens(stats.cacheRead) + ' cached'); + if (stats.cacheWrite) parts.push(fmtTokens(stats.cacheWrite) + ' new'); + tokenText += ' (' + parts.join(', ') + ')'; + } + tokenText += ' → ' + fmtTokens(stats.completionTokens); + header.appendChild(el('span', {className: 'span-tokens'}, tokenText)); + } + if (stats.cost > 0) { + header.appendChild(el('span', {className: 'span-cost'}, '$' + stats.cost.toFixed(2))); + } + } + + header.appendChild(el('span', {className: 'span-duration'}, fmtDuration(span.start_time, span.end_time))); + row.appendChild(header); + + const detail = extractDetail(attrs, span.name); + if (detail) { + const detailDiv = el('div', {className: 'span-detail'}); + detailDiv.appendChild(detail); + row.appendChild(detailDiv); + } + + const attrHtml = renderAttrs(attrs); + if (attrHtml) row.appendChild(attrHtml); + + return row; +} + +// ============================================================ +// Span Detail Extraction +// ============================================================ + +function extractDetail(attrs, spanName) { + const kind = getVal(attrs['openinference.span.kind']); + + if (kind === 'LLM' || (!kind && spanName.endsWith('ChatModel'))) { + const frag = document.createDocumentFragment(); + let found = false; + + let i = 0; + while (true) { + const ctype = getVal(attrs['llm.output_messages.0.message.contents.' + i + '.message_content.type']); + if (ctype == null) break; + if (ctype === 'reasoning') { + const text = getVal(attrs['llm.output_messages.0.message.contents.' + i + '.message_content.text']); + if (text) { + frag.appendChild(lazyDetails('reasoning (' + text.length + ' chars)', + () => el('div', {className: 'detail-reasoning', textContent: text}), true)); + found = true; + } + } else if (ctype === 'text') { + const text = getVal(attrs['llm.output_messages.0.message.contents.' + i + '.message_content.text']); + if (text) { + frag.appendChild(el('div', {className: 'detail-text', textContent: text})); + found = true; + } + } + i++; + } + + const toolCalls = []; + i = 0; + while (true) { + const name = getVal(attrs['llm.output_messages.0.message.tool_calls.' + i + '.tool_call.function.name']); + if (name == null) break; + const args = getVal(attrs['llm.output_messages.0.message.tool_calls.' + i + '.tool_call.function.arguments']) || ''; + toolCalls.push({name, args}); + i++; + } + if (toolCalls.length > 0) { + const label = toolCalls.map(tc => tc.name).join(', '); + frag.appendChild(lazyDetails('tool calls: ' + label, () => { + const f = document.createDocumentFragment(); + for (const tc of toolCalls) { + const truncated = tc.args.length > 500 ? tc.args.slice(0, 500) + '...' : tc.args; + const toolDiv = el('div', {className: 'detail-tool-call'}, + el('div', {className: 'detail-tool-name', textContent: tc.name})); + if (truncated) { + toolDiv.appendChild(el('pre', {textContent: truncated})); + } + f.appendChild(toolDiv); + } + return f; + })); + found = true; + } + + return found ? frag : null; + } + + if (spanName === 'error') { + const output = getVal(attrs['output.value']); + if (output) { + const truncated = String(output).slice(0, 1000); + return el('div', {className: 'detail-error', textContent: truncated}); + } + return null; + } + + if (kind === 'TOOL' && spanName === 'final_answer') { + const inputVal = getVal(attrs['input.value']); + if (inputVal == null) return null; + let content; + try { + const parsed = JSON.parse(inputVal); + const args = parsed.input || parsed; + if (typeof args.response === 'string') { + try { content = JSON.stringify(JSON.parse(args.response), null, 2); } + catch (e) { content = args.response; } + } else { + content = JSON.stringify(args, null, 2); + } + } catch (e) { + content = String(inputVal); + } + return lazyDetails('content (' + content.length + ' chars)', + () => el('pre', {className: 'detail-tool-io', textContent: content}), true); + } + + if (kind === 'TOOL' && spanName === 'run_shell_command') { + const frag = document.createDocumentFragment(); + const outputVal = getVal(attrs['output.value']); + if (outputVal == null) return null; + let result; + try { result = JSON.parse(outputVal); } catch (e) { + frag.appendChild(el('pre', {className: 'detail-tool-io', textContent: outputVal})); + return frag; + } + if (result.stdout) { + frag.appendChild(lazyDetails('stdout (' + result.stdout.length + ' chars)', + () => el('pre', {className: 'detail-tool-io', textContent: result.stdout}), true)); + } + if (result.stderr) { + frag.appendChild(lazyDetails('stderr (' + result.stderr.length + ' chars)', + () => el('pre', {className: 'detail-error', textContent: result.stderr}), true)); + } + if (result.exit_code !== 0) { + frag.appendChild(el('div', {className: 'detail-error', textContent: 'exit code: ' + result.exit_code})); + } + return frag.childNodes.length ? frag : null; + } + + if (kind === 'TOOL') { + const frag = document.createDocumentFragment(); + let found = false; + + const inputVal = getVal(attrs['input.value']); + if (inputVal != null) { + let pretty; + try { + const parsed = JSON.parse(inputVal); + pretty = JSON.stringify(parsed.input || parsed, null, 2); + } catch (e) { + pretty = String(inputVal); + } + frag.appendChild(lazyDetails('input (' + pretty.length + ' chars)', + () => el('pre', {className: 'detail-tool-io', textContent: pretty}))); + found = true; + } + const outputVal = getVal(attrs['output.value']); + if (outputVal != null) { + const str = String(outputVal); + const isError = str.startsWith('ToolError'); + const cls = isError ? 'detail-error' : 'detail-tool-io'; + let pretty = isError ? str.replace(/\n\s*Context: .*/g, '') : str; + if (!isError) { + try { pretty = JSON.stringify(JSON.parse(str), null, 2); } catch (e) {} + } + frag.appendChild(lazyDetails((isError ? 'error' : 'output') + ' (' + pretty.length + ' chars)', + () => el('pre', {className: cls, textContent: pretty}), true)); + found = true; + } + return found ? frag : null; + } + + return null; +} + +function renderAttrs(attrs) { + if (!attrs) return null; + const keys = Object.keys(attrs).sort(); + if (keys.length === 0) return null; + + const lines = keys.map(k => { + const v = getVal(attrs[k]); + return '' + escHtml(k) + ': ' + escHtml(String(v)); + }).join('\n'); + + return lazyDetails(keys.length + ' attributes', + () => el('pre', {innerHTML: lines})); +} + +// ============================================================ +// Agent Sidebar +// ============================================================ + +function collectAgents(nodes, depth) { + const agents = []; + for (const node of nodes) { + const attrs = node.attributes || {}; + const kind = getVal(attrs['openinference.span.kind']) || ''; + const isAgent = kind === 'AGENT' || kind === 'CHAIN' || node._placeholder + || node.name.endsWith('Workflow') || node.name.endsWith('Agent') || node.name.endsWith('Analyst'); + if (isAgent) { + agents.push({ + span_id: node.span_id, + name: node.name, + depth: depth, + status_code: node.status_code, + end_time: node.end_time, + children: node.children ? collectAgents(node.children, depth + 1) : [], + }); + } else if (node.children && node.children.length > 0) { + agents.push(...collectAgents(node.children, depth)); + } + } + return agents; +} + +let sidebarScrollHandler = null; +let sidebarScrollLocked = false; +let sidebarScrollLockTimer = null; + +function updateSidebarActive(nav, spanIds) { + if (sidebarScrollLocked) return; + const headerBottom = 50; + let active = null; + for (const id of spanIds) { + const elem = document.getElementById('span-' + id); + if (!elem) continue; + if (elem.getBoundingClientRect().top <= headerBottom) active = id; + } + if (active == null && spanIds.length > 0) { + active = spanIds[0]; + } + nav.querySelectorAll('.sidebar-item').forEach(item => { + item.classList.toggle('active', item.getAttribute('data-span-id') === active); + }); +} + +function setSidebarActive(nav, spanId) { + nav.querySelectorAll('.sidebar-item').forEach(item => { + item.classList.toggle('active', item.getAttribute('data-span-id') === spanId); + }); +} + +function renderSidebar(agents) { + const nav = el('nav', {className: 'trace-sidebar', id: 'trace-sidebar'}); + + function addItems(list, depth) { + for (const agent of list) { + const sc = statusClass(agent.status_code); + const item = el('div', { + className: 'sidebar-item', + style: 'padding-left: ' + (8 + depth * 16) + 'px', + 'data-span-id': agent.span_id, + onClick: () => { + const target = document.getElementById('span-' + agent.span_id); + if (target) { + setSidebarActive(nav, agent.span_id); + sidebarScrollLocked = true; + if (sidebarScrollLockTimer) clearTimeout(sidebarScrollLockTimer); + target.scrollIntoView({behavior: 'smooth', block: 'start'}); + } + }, + }); + item.appendChild(el('span', {className: 'status-dot ' + sc})); + item.appendChild(el('span', {textContent: agent.name})); + nav.appendChild(item); + if (agent.children.length > 0) { + addItems(agent.children, depth + 1); + } + } + } + + addItems(agents, 0); + + const spanIds = []; + function gatherIds(list) { + for (const a of list) { + spanIds.push(a.span_id); + gatherIds(a.children); + } + } + gatherIds(agents); + + if (sidebarScrollHandler) window.removeEventListener('scroll', sidebarScrollHandler); + sidebarScrollHandler = () => { + if (sidebarScrollLocked) { + if (sidebarScrollLockTimer) clearTimeout(sidebarScrollLockTimer); + sidebarScrollLockTimer = setTimeout(() => { sidebarScrollLocked = false; }, 150); + return; + } + updateSidebarActive(nav, spanIds); + }; + window.addEventListener('scroll', sidebarScrollHandler, {passive: true}); + requestAnimationFrame(() => updateSidebarActive(nav, spanIds)); + + return nav; +} + +// ============================================================ +// Auto-scroll +// ============================================================ + +let autoScroll = true; +let jumpBtn = null; + +function setupAutoScroll() { + autoScroll = true; + window.addEventListener('scroll', onScroll); +} + +function onScroll() { + if (state.view !== 'trace') { + window.removeEventListener('scroll', onScroll); + return; + } + const nearBottom = (window.innerHeight + window.scrollY) >= (document.body.scrollHeight - 100); + if (nearBottom) { + autoScroll = true; + if (jumpBtn) { jumpBtn.remove(); jumpBtn = null; } + } else { + autoScroll = false; + } +} + +function maybeAutoScroll() { + if (autoScroll) { + window.scrollTo({top: document.body.scrollHeight, behavior: 'smooth'}); + } else if (!jumpBtn) { + jumpBtn = el('button', { + className: 'jump-bottom', + onClick: () => { + autoScroll = true; + window.scrollTo({top: document.body.scrollHeight, behavior: 'smooth'}); + if (jumpBtn) { jumpBtn.remove(); jumpBtn = null; } + }, + }, '↓ jump to bottom'); + document.body.appendChild(jumpBtn); + } +} + +// ============================================================ +// Issues View +// ============================================================ + +async function renderIssues(container) { + container.appendChild(el('div', {className: 'loading'}, 'loading issues...')); + + try { + const data = await api.issues(); + const issues = data.issues || []; + container.innerHTML = ''; + container.appendChild(el('div', {className: 'view-title'}, 'Issues (' + issues.length + ')')); + + if (issues.length === 0) { + container.appendChild(el('div', {className: 'empty-state'}, 'No issues found.')); + return; + } + + const list = el('div', {className: 'issue-list'}); + for (const issue of issues) { + list.appendChild(el('a', { + className: 'issue-row', + href: '#/issues/' + encodeURIComponent(issue), + }, issue)); + } + container.appendChild(list); + setStatus(issues.length + ' issues'); + } catch (e) { + container.innerHTML = ''; + container.appendChild(el('div', {className: 'error-banner'}, 'Failed to load issues: ' + e.message)); + } + + startPolling(() => refreshIssues(container), 30000); +} + +async function refreshIssues(container) { + try { + const data = await api.issues(); + const issues = data.issues || []; + container.innerHTML = ''; + container.appendChild(el('div', {className: 'view-title'}, 'Issues (' + issues.length + ')')); + if (issues.length === 0) { + container.appendChild(el('div', {className: 'empty-state'}, 'No issues found.')); + } else { + const list = el('div', {className: 'issue-list'}); + for (const issue of issues) { + list.appendChild(el('a', { + className: 'issue-row', + href: '#/issues/' + encodeURIComponent(issue), + }, issue)); + } + container.appendChild(list); + } + setStatus(issues.length + ' issues'); + } catch (e) {} +} + +// ============================================================ +// Issue Detail View +// ============================================================ + +function sinceNanos(seconds) { + return (Date.now() - seconds * 1000) * 1e6; +} + +async function renderIssueDetail(container, issue) { + container.appendChild(el('div', {className: 'loading'}, 'loading traces for ' + issue + '...')); + + const filtersBar = el('div', {className: 'filters'}); + const sinceSelect = el('select', {className: 'filter-select', onChange: (e) => { + state.issueFilterSince = parseInt(e.target.value); + refreshIssueDetail(container, issue); + }}); + for (const [label, val] of [['1h', 3600], ['3h', 10800], ['12h', 43200], ['24h', 86400], ['3d', 259200], ['7d', 604800], ['30d', 2592000], ['all', 0]]) { + const opt = el('option', {value: val}, label); + if (val === state.issueFilterSince) opt.selected = true; + sinceSelect.appendChild(opt); + } + filtersBar.appendChild(el('span', {className: 'filter-label'}, 'since:')); + filtersBar.appendChild(sinceSelect); + + try { + const opts = {}; + if (state.issueFilterSince > 0) opts.since = sinceNanos(state.issueFilterSince); + const data = await api.spans(issue, opts); + const spans = data.spans || []; + container.innerHTML = ''; + container.appendChild(el('a', {className: 'back-link', href: '#/issues'}, '← issues')); + container.appendChild(filtersBar); + container.appendChild(el('div', {className: 'view-title'}, issue)); + + if (spans.length === 0) { + container.appendChild(el('div', {className: 'empty-state'}, 'No traces found.')); + return; + } + + renderIssueTraces(container, issue, spans); + } catch (e) { + container.innerHTML = ''; + container.appendChild(el('a', {className: 'back-link', href: '#/issues'}, '← issues')); + container.appendChild(filtersBar); + container.appendChild(el('div', {className: 'error-banner'}, 'Failed to load: ' + e.message)); + } + + startPolling(() => refreshIssueDetail(container, issue), 30000); +} + +function traceWorkflowName(spans) { + const root = spans.find(s => !s.parent_span_id || s.parent_span_id === ''); + if (root) return root.name; + for (const s of spans) { + const wf = getVal((s.attributes || {})['workflow.name']); + if (wf) return wf[0].toUpperCase() + wf.slice(1) + 'Workflow'; + } + return spans[0]?.name || 'trace'; +} + +function renderIssueTraces(container, issue, spans) { + const byTrace = new Map(); + for (const s of spans) { + if (!byTrace.has(s.trace_id)) byTrace.set(s.trace_id, []); + byTrace.get(s.trace_id).push(s); + } + + const traceIds = [...byTrace.keys()]; + traceIds.sort((a, b) => { + const aStart = byTrace.get(a)[0].start_time || 0; + const bStart = byTrace.get(b)[0].start_time || 0; + return bStart - aStart; + }); + + for (const tid of traceIds) { + const traceSpans = byTrace.get(tid); + const label = traceWorkflowName(traceSpans); + const first = traceSpans[0]; + const errors = traceSpans.filter(s => s.status_code === 2).length; + const sc = errors > 0 ? 'error' : statusClass(first.status_code); + + const group = el('div', {className: 'trace-group'}); + const header = el('div', {className: 'trace-group-header', onClick: () => { + location.hash = '#/trace/' + encodeURIComponent(issue) + '/' + tid; + }}); + header.appendChild(el('span', {className: 'status-dot ' + sc})); + header.appendChild(el('span', {}, label + ' — ' + tid.slice(0, 16) + '…')); + header.appendChild(el('span', {className: 'trace-card-time'}, fmtAgo(first.start_time))); + header.appendChild(el('span', {}, traceSpans.length + ' spans')); + if (errors > 0) header.appendChild(el('span', {className: 'error-count'}, errors + ' errors')); + group.appendChild(header); + container.appendChild(group); + } + + setStatus(traceIds.length + ' traces, ' + spans.length + ' spans'); +} + +async function refreshIssueDetail(container, issue) { + try { + const opts = {}; + if (state.issueFilterSince > 0) opts.since = sinceNanos(state.issueFilterSince); + const data = await api.spans(issue, opts); + const spans = data.spans || []; + const filters = container.querySelector('.filters'); + const backLink = container.querySelector('.back-link'); + container.innerHTML = ''; + if (backLink) container.appendChild(backLink); + if (filters) container.appendChild(filters); + container.appendChild(el('div', {className: 'view-title'}, issue)); + + if (spans.length === 0) { + container.appendChild(el('div', {className: 'empty-state'}, 'No traces found.')); + setStatus('0 traces'); + return; + } + + renderIssueTraces(container, issue, spans); + } catch (e) {} +} + +// ============================================================ +// Keyboard shortcuts +// ============================================================ + +document.addEventListener('keydown', (e) => { + if (e.target.tagName === 'INPUT' || e.target.tagName === 'SELECT' || e.target.tagName === 'TEXTAREA') return; + if (e.key === 'Escape') { + if (state.view === 'trace') location.hash = '#/'; + else if (state.view === 'issue') location.hash = '#/issues'; + } + if (e.key === 'r' && !e.ctrlKey && !e.metaKey) { + route(); + } +}); + +// ============================================================ +// Init +// ============================================================ + +renderHeader(); +route(); diff --git a/trace_server/static/index.html b/trace_server/static/index.html new file mode 100644 index 000000000..850308cc2 --- /dev/null +++ b/trace_server/static/index.html @@ -0,0 +1,18 @@ + + + + + + Ymir traces + + + + + + +
+ + + diff --git a/trace_server/static/style.css b/trace_server/static/style.css new file mode 100644 index 000000000..d82bae03f --- /dev/null +++ b/trace_server/static/style.css @@ -0,0 +1,596 @@ +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + +:root { + --bg-primary: #fafafa; + --bg-secondary: #f0f0f0; + --bg-tertiary: #e4e4e4; + --fg-primary: #1a1a1a; + --fg-secondary: #555; + --fg-muted: #999; + --border: #ddd; + --accent: #0066cc; + --accent-hover: #0052a3; + --status-ok: #16a34a; + --status-error: #dc2626; + --status-unset: #9ca3af; + --reasoning-bg: #fdf8f0; + --reasoning-border: #c8a060; + --tool-bg: #f0f5f8; + --tool-border: #7a9db5; + --error-bg: #fef2f2; + --error-border: #ef4444; + --pre-bg: #f5f5f5; + --card-hover: rgba(0, 102, 204, 0.04); + --header-bg: #1a1a1a; + --header-fg: #e0e0e0; +} + +html.dark { + --bg-primary: #0d0d0d; + --bg-secondary: #161616; + --bg-tertiary: #222; + --fg-primary: #d4d4d4; + --fg-secondary: #999; + --fg-muted: #666; + --border: #2a2a2a; + --accent: #58a6ff; + --accent-hover: #79b8ff; + --status-ok: #3fb950; + --status-error: #f85149; + --status-unset: #6b7280; + --reasoning-bg: #1a1408; + --reasoning-border: #8a6830; + --tool-bg: #0c1418; + --tool-border: #4a7a90; + --error-bg: #1f0a0a; + --error-border: #ef4444; + --pre-bg: #1a1a1a; + --card-hover: rgba(88, 166, 255, 0.06); + --header-bg: #0a0a0a; + --header-fg: #d4d4d4; +} + +body { + font-family: "JetBrains Mono", "Fira Code", "Cascadia Code", "SF Mono", "Consolas", "Liberation Mono", monospace; + font-size: 13px; + line-height: 1.5; + background: var(--bg-primary); + color: var(--fg-primary); + min-height: 100vh; +} + +* { + scrollbar-width: thin; + scrollbar-color: var(--bg-tertiary) transparent; +} + +*:hover { + scrollbar-color: var(--fg-muted) transparent; +} + +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } + +/* Header */ +#header { + position: sticky; + top: 0; + z-index: 100; + background: var(--header-bg); + color: var(--header-fg); + padding: 8px 16px; + display: flex; + align-items: center; + gap: 16px; + border-bottom: 1px solid var(--border); +} + +.header-logo { + font-size: 15px; + font-weight: bold; + color: var(--header-fg); + text-decoration: none; +} + +.header-logo:hover { text-decoration: none; } + +.header-avatar { + width: 22px; + height: 22px; + border-radius: 50%; + margin-right: 8px; + vertical-align: middle; +} + +.cursor { + animation: blink 1s step-end infinite; +} + +@keyframes blink { + 50% { opacity: 0; } +} + +.header-nav { + display: flex; + gap: 12px; +} + +.header-nav a { + color: var(--fg-muted); + font-size: 12px; + padding: 2px 6px; +} + +.header-nav a.active { + color: var(--header-fg); + background: rgba(255, 255, 255, 0.1); + border-radius: 2px; +} + +.header-spacer { flex: 1; } + +.header-status { + font-size: 11px; + color: var(--fg-muted); +} + +.header-btn { + background: none; + border: 1px solid rgba(255, 255, 255, 0.2); + color: var(--header-fg); + font-family: inherit; + font-size: 11px; + padding: 2px 8px; + cursor: pointer; + border-radius: 2px; +} + +.header-btn:hover { + background: rgba(255, 255, 255, 0.1); +} + +/* Main content */ +#app { + max-width: 1200px; + margin: 0 auto; + padding: 16px; +} + +#app:has(.trace-layout) { + max-width: 1500px; +} + +/* Trace cards grid */ +.trace-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(340px, 1fr)); + gap: 8px; +} + +.trace-card { + background: var(--bg-secondary); + border: 1px solid var(--border); + border-left: 3px solid var(--status-unset); + padding: 10px 12px; + cursor: pointer; + transition: background 0.15s; +} + +.trace-card:hover { + background: var(--card-hover); +} + +.trace-card.status-ok { border-left-color: var(--status-ok); } +.trace-card.status-error { border-left-color: var(--status-error); } + +.trace-card-header { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 4px; +} + +.trace-card-workflow { + font-weight: bold; + font-size: 13px; +} + +.trace-card-time { + margin-left: auto; + font-size: 11px; + color: var(--fg-muted); +} + +.trace-card-issues { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin-top: 6px; +} + +.issue-badge { + font-size: 11px; + padding: 1px 6px; + background: var(--bg-tertiary); + border: 1px solid var(--border); + border-radius: 2px; + color: var(--accent); + text-decoration: none; +} + +.issue-badge:hover { text-decoration: none; background: var(--card-hover); } + +.trace-card-meta { + display: flex; + gap: 12px; + margin-top: 6px; + font-size: 11px; + color: var(--fg-secondary); +} + +.error-count { + color: var(--status-error); + font-weight: bold; +} + +/* Status dot */ +.status-dot { + display: inline-block; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--status-unset); +} + +.status-dot.ok { background: var(--status-ok); } +.status-dot.error { background: var(--status-error); } + +/* Filters bar */ +.filters { + display: flex; + gap: 8px; + margin-bottom: 12px; + align-items: center; + flex-wrap: wrap; +} + +.filter-select, .filter-input { + font-family: inherit; + font-size: 12px; + padding: 3px 8px; + background: var(--bg-secondary); + color: var(--fg-primary); + border: 1px solid var(--border); + border-radius: 2px; +} + +.filter-label { + font-size: 11px; + color: var(--fg-muted); +} + +/* Trace detail view */ +.trace-detail-header { + margin-bottom: 12px; + padding-bottom: 8px; + border-bottom: 1px solid var(--border); +} + +.trace-detail-header h1 { + font-size: 16px; + font-weight: bold; + margin-bottom: 4px; +} + +.trace-detail-meta { + display: flex; + gap: 16px; + font-size: 12px; + color: var(--fg-secondary); + flex-wrap: wrap; +} + +.back-link { + font-size: 12px; + margin-bottom: 8px; + display: inline-block; +} + +/* Span list */ +.span-list { + display: flex; + flex-direction: column; + gap: 2px; +} + +.span-row { + background: var(--bg-secondary); + border: 1px solid var(--border); + border-left: 3px solid var(--fg-muted); + padding: 6px 10px; + font-size: 12px; +} + +.span-row.kind-llm { border-left-color: var(--accent); } +.span-row.kind-tool { border-left-color: var(--tool-border); } +.span-row.kind-agent { border-left-color: #a855f7; } +.span-row.has-error { border-left-color: var(--error-border); } + +.span-row-header { + display: flex; + align-items: center; + gap: 8px; +} + +.span-name { + font-weight: bold; + font-size: 12px; +} + +.span-args { + font-weight: normal; + color: var(--fg-primary); +} + +.span-kind { + font-size: 10px; + padding: 0 4px; + border: 1px solid var(--border); + border-radius: 2px; + color: var(--fg-muted); + text-transform: uppercase; +} + +.span-duration { + margin-left: auto; + font-size: 11px; + color: var(--fg-muted); +} + +.span-status { + font-size: 11px; +} + +.span-status.ok { color: var(--status-ok); } +.span-status.error { color: var(--status-error); font-weight: bold; } + +.span-tokens { + font-size: 11px; + color: var(--fg-muted); +} + +.span-cost { + font-size: 11px; + color: var(--fg-muted); +} + +/* Span detail blocks */ +.span-detail { margin-top: 6px; } + +.detail-reasoning { + background: var(--reasoning-bg); + border-left: 3px solid var(--reasoning-border); + padding: 6px 10px; + margin: 4px 0; + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; + max-height: 400px; + overflow: auto; +} + +.detail-text { + padding: 6px 10px; + margin: 4px 0; + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; +} + +.detail-tool-call { + background: var(--tool-bg); + border-left: 3px solid var(--tool-border); + padding: 6px 10px; + margin: 4px 0; + font-size: 12px; + word-break: break-all; +} + +.detail-tool-name { + font-weight: bold; + font-size: 12px; + margin-bottom: 2px; +} + +.detail-tool-io { + background: var(--tool-bg); + border-left: 3px solid var(--tool-border); + padding: 6px 10px; + margin: 4px 0; + font-size: 11px; + max-height: 300px; + overflow: auto; + white-space: pre-wrap; + word-break: break-word; +} + +.detail-error { + background: var(--error-bg); + border-left: 3px solid var(--error-border); + padding: 6px 10px; + margin: 4px 0; + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; +} + +details { margin-top: 4px; } +details summary { + cursor: pointer; + color: var(--fg-muted); + font-size: 11px; + user-select: none; +} +details summary:hover { color: var(--fg-secondary); } + +pre { + margin: 4px 0; + font-size: 11px; + max-height: 300px; + overflow: auto; + background: var(--pre-bg); + padding: 8px; + border-radius: 2px; + white-space: pre-wrap; + word-break: break-word; +} + +.attr-key { color: #a855f7; } +html.dark .attr-key { color: #c9a0ff; } + +/* Issues list */ +.issue-list { + display: flex; + flex-direction: column; + gap: 2px; +} + +.issue-row { + display: block; + padding: 8px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border); + font-size: 13px; +} + +.issue-row:hover { background: var(--card-hover); } + +/* Loading & empty states */ +.loading { + text-align: center; + padding: 32px; + color: var(--fg-muted); + font-size: 13px; +} + +.empty-state { + text-align: center; + padding: 48px 16px; + color: var(--fg-muted); +} + +.error-banner { + background: var(--error-bg); + border: 1px solid var(--error-border); + padding: 10px 14px; + margin-bottom: 12px; + font-size: 12px; + color: var(--status-error); +} + +/* Jump to bottom */ +.jump-bottom { + position: fixed; + bottom: 16px; + right: 16px; + background: var(--accent); + color: #fff; + border: none; + font-family: inherit; + font-size: 11px; + padding: 6px 12px; + cursor: pointer; + border-radius: 2px; + z-index: 50; +} + +.jump-bottom:hover { background: var(--accent-hover); } + +.hidden { display: none !important; } + +/* View title */ +.view-title { + font-size: 14px; + font-weight: bold; + margin-bottom: 12px; + color: var(--fg-primary); +} + +/* Issue detail: trace groups */ +.trace-group { + margin-bottom: 16px; +} + +.trace-group-header { + display: flex; + align-items: center; + gap: 8px; + padding: 6px 10px; + background: var(--bg-tertiary); + border: 1px solid var(--border); + font-size: 12px; + cursor: pointer; + user-select: none; +} + +.trace-group-header:hover { background: var(--card-hover); } + +/* Trace layout with sidebar */ +.trace-layout { + display: grid; + grid-template-columns: 220px 1fr; + gap: 0 16px; + align-items: start; + max-width: 1600px; + margin: 0 auto; +} + +.trace-sidebar { + position: sticky; + top: 42px; + max-height: calc(100vh - 50px); + overflow-y: auto; + font-size: 12px; + padding: 8px 0; + border-right: 1px solid var(--border); +} + +.sidebar-item { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 8px; + cursor: pointer; + color: var(--fg-secondary); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + border-left: 2px solid transparent; +} + +.sidebar-item:hover { + background: var(--card-hover); + color: var(--fg-primary); +} + +.sidebar-item.active { + color: var(--fg-primary); + font-weight: bold; + border-left-color: var(--accent); +} + +.trace-main { + min-width: 0; +} + +/* Responsive */ +@media (max-width: 800px) { + .trace-layout { grid-template-columns: 1fr; } + .trace-sidebar { display: none; } +} + +@media (max-width: 600px) { + #app { padding: 8px; } + .trace-grid { grid-template-columns: 1fr; } + .trace-detail-meta { flex-direction: column; gap: 4px; } +} diff --git a/trace_server/static/ymir-avatar.png b/trace_server/static/ymir-avatar.png new file mode 100644 index 000000000..6ef82dcd5 Binary files /dev/null and b/trace_server/static/ymir-avatar.png differ diff --git a/ymir/agents/observability.py b/ymir/agents/observability.py index 5bb531c20..4341b7c92 100644 --- a/ymir/agents/observability.py +++ b/ymir/agents/observability.py @@ -1,5 +1,6 @@ import atexit import contextlib +import threading import sentry_sdk from openinference.instrumentation.beeai import BeeAIInstrumentor @@ -11,10 +12,14 @@ from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.sdk.trace.export import BatchSpanProcessor -from ymir.common.logging_setup import current_jira_issue +from ymir.common.logging_setup import current_jira_issue, current_workflow class AgentSpanProcessor(SpanProcessor): + def __init__(self) -> None: + self._agent_by_span: dict[int, str] = {} + self._lock = threading.Lock() + def set_jira_issue(self, jira_issue: str | None) -> None: current_jira_issue.set(jira_issue) @@ -39,20 +44,38 @@ def start_transaction( transaction.set_data("workflow", workflow) transaction.set_data("jira_issue", jira_issue) - token = current_jira_issue.set(jira_issue) + issue_token = current_jira_issue.set(jira_issue) + workflow_token = current_workflow.set(workflow) try: yield finally: - current_jira_issue.reset(token) + current_jira_issue.reset(issue_token) + current_workflow.reset(workflow_token) def on_start(self, span: Span, parent_context: Context | None = None) -> None: if span.is_recording(): jira_issue = current_jira_issue.get() if jira_issue: span.set_attribute("jira.issue", jira_issue) + workflow = current_workflow.get() + if workflow: + span.set_attribute("workflow.name", workflow) + agent = None + if span.name.endswith(("Agent", "Analyst")): + agent = span.name + if not agent and parent_context: + parent = trace_api.get_current_span(parent_context) + if parent and parent.context.span_id: + with self._lock: + agent = self._agent_by_span.get(parent.context.span_id) + if agent: + span.set_attribute("agent.name", agent) + with self._lock: + self._agent_by_span[span.context.span_id] = agent def on_end(self, span: ReadableSpan) -> None: - pass + with self._lock: + self._agent_by_span.pop(getattr(span.context, "span_id", None), None) def shutdown(self) -> None: pass diff --git a/ymir/common/logging_setup.py b/ymir/common/logging_setup.py index 23c6f3715..06b1d18d8 100644 --- a/ymir/common/logging_setup.py +++ b/ymir/common/logging_setup.py @@ -10,6 +10,7 @@ LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" current_jira_issue: ContextVar[str | None] = ContextVar("current_jira_issue", default=None) +current_workflow: ContextVar[str | None] = ContextVar("current_workflow", default=None) _buffered_handler: "BufferedTaskHandler | None" = None