diff --git a/README.md b/README.md index cb6dbb23..334ced0b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Use dreb if you want a coding agent that can run against direct APIs, coding sub - **A capable terminal workspace.** The TUI supports slash commands, file references with `@`, path completion, image paste/drag, bash shortcuts, hotkeys, settings, model cycling, steering/follow-up queues while the agent is working, token/cost/context status, custom themes, and extension-provided UI surfaces. Transcript prose, code, tool output, and agent results use terminal soft-wrap so copying from scrollback keeps long logical lines intact instead of injecting hard newlines. - **Optional local companion.** [`/buddy`](packages/coding-agent/docs/buddy.md) hatches an Ollama-powered terminal companion with persistent state, generated personality/backstory, event reactions, idle quips, name-call responses, pet/reroll/stats commands, and a sidebar presence while you work. - **Codebase and web understanding.** dreb includes file, grep/find/ls, bash, web search/fetch, task tracking, skill invocation, and semantic `search`. Semantic search uses AST-aware chunks, embeddings, POEM ranking, memory indexing, and also ships as [`@dreb/semantic-search`](packages/semantic-search/) with an MCP server for other harnesses. The semantic search package requires Node.js 22+. -- **Detailed usage tracking and performance logging.** dreb records per-session token usage, cost, context-window utilization, and rolling tokens-per-second performance in a local JSONL log (`~/.dreb/agent/performance.jsonl`). This data stays on your machine and can be queried via the TUI footer, Telegram `/stats`, or RPC for personal analytics and model comparison. +- **Detailed usage tracking and performance logging.** dreb records per-session token usage, cost, context-window utilization, and rolling tokens-per-second performance in a local JSONL log (`~/.dreb/agent/performance.jsonl`). This data stays on your machine; the TUI footer, dashboard session details, Telegram `/stats`, and RPC share the same latest-100 median with long-term delta for personal analytics and model comparison. - **Safety and reliability primitives.** Recent dreb-specific hardening includes secret output scrubbing, sensitive-file guards, destructive-command guards, resource diagnostics surfaced in-session, warning propagation, rate-limited web search across parallel subagents, and JSON/RPC protocol hardening. Dropped provider streams are retried (discarding the partial), and responses truncated at the model's output-token limit are retried with a larger token budget — failing loudly rather than returning a silently empty or truncated result. - **Multiple interfaces.** Run dreb as an interactive TUI, print/headless CLI, JSON event stream, RPC process, embedded [SDK](packages/coding-agent/docs/sdk.md), [web dashboard](packages/coding-agent/docs/dashboard.md), or [Telegram bot](packages/telegram/). diff --git a/package-lock.json b/package-lock.json index 16d0bbd7..01a7445f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dreb", - "version": "2.55.1", + "version": "2.55.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dreb", - "version": "2.55.1", + "version": "2.55.2", "workspaces": [ "packages/*", "packages/coding-agent/examples/extensions/with-deps", @@ -10955,7 +10955,7 @@ }, "packages/agent": { "name": "@dreb/agent-core", - "version": "2.55.1", + "version": "2.55.2", "license": "MIT", "dependencies": { "@dreb/ai": "*" @@ -10984,7 +10984,7 @@ }, "packages/ai": { "name": "@dreb/ai", - "version": "2.55.1", + "version": "2.55.2", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.73.0", @@ -11040,7 +11040,7 @@ }, "packages/coding-agent": { "name": "@dreb/coding-agent", - "version": "2.55.1", + "version": "2.55.2", "license": "MIT", "dependencies": { "@dreb/agent-core": "*", @@ -11169,7 +11169,7 @@ }, "packages/dashboard": { "name": "@dreb/dashboard", - "version": "2.55.1", + "version": "2.55.2", "license": "MIT", "dependencies": { "@dreb/coding-agent": "*", @@ -11401,7 +11401,7 @@ }, "packages/semantic-search": { "name": "@dreb/semantic-search", - "version": "2.55.1", + "version": "2.55.2", "license": "MIT", "dependencies": { "@huggingface/transformers": "^4.0.1", @@ -11450,7 +11450,7 @@ }, "packages/telegram": { "name": "@dreb/telegram", - "version": "2.55.1", + "version": "2.55.2", "license": "MIT", "dependencies": { "@dreb/coding-agent": "*", @@ -11483,7 +11483,7 @@ }, "packages/tui": { "name": "@dreb/tui", - "version": "2.55.1", + "version": "2.55.2", "license": "MIT", "dependencies": { "@types/mime-types": "^2.1.4", diff --git a/package.json b/package.json index e87cd7f0..897911bb 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "node": "22.x" }, "packageManager": "npm@11.5.1", - "version": "2.55.1", + "version": "2.55.2", "dependencies": { "@dreb/coding-agent": "*", "@mariozechner/jiti": "^2.6.5", diff --git a/packages/agent/package.json b/packages/agent/package.json index 881069a1..1e7097a3 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@dreb/agent-core", - "version": "2.55.1", + "version": "2.55.2", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "type": "module", "main": "./dist/index.js", diff --git a/packages/ai/package.json b/packages/ai/package.json index b8bd28e4..feebce9b 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@dreb/ai", - "version": "2.55.1", + "version": "2.55.2", "description": "Unified LLM API with automatic model discovery and provider configuration", "type": "module", "main": "./dist/index.js", diff --git a/packages/coding-agent/README.md b/packages/coding-agent/README.md index 3e25cb76..017cc7f4 100644 --- a/packages/coding-agent/README.md +++ b/packages/coding-agent/README.md @@ -81,7 +81,7 @@ Or use a custom provider (corporate proxy, Bedrock, etc.) — see [Custom provid Then just talk to dreb. All 13 standard built-in tools are enabled by default: `read`, `write`, `edit`, `bash`, `grep`, `find`, `ls`, `web_search`, `web_fetch`, `subagent`, `wait`, `watch_github_ci`, and `ask_user`. Use `--tools` to restrict to a subset (e.g., `--tools read,grep,find,ls` for read-only). Three additional tools — `search`, `skill`, and `tasks_update` — are always active regardless of `--tools`. `suggest_next` is active by default but excluded when `--tools` is specified. The model uses these to fulfill your requests. Add capabilities via [skills](#skills), [prompt templates](#prompt-templates), [extensions](#extensions), or [packages](#packages). -**Also available:** [`@dreb/telegram`](https://www.npmjs.com/package/@dreb/telegram) — run dreb as a Telegram bot with live tool status and visible results for user-facing tools (`npm install -g @dreb/telegram`). [`@dreb/dashboard`](https://www.npmjs.com/package/@dreb/dashboard) — run `dreb dashboard` for a browser UI with fleet overview, full chat steering, generic fail-closed built-in slash-command discovery and execution, inline provider/API failures with partial output preserved, sanitized raster tool images plus sent user uploads retained as bounded transcript previews by default, a bounded all-agent subagent panel with drill-in, host file browser, dreb memory editor with exact-revision saves and automatic index cleanup on delete, curated appearance themes (per-browser light/dark), and Tailscale/rotating-code pairing (`npm install -g @dreb/dashboard`; see [docs/dashboard.md](docs/dashboard.md)). Tool images cross browser-facing transport as content-addressed references; browser-local Settings offers placeholders, bounded previews, or informed-opt-in originals, with size disclosure and confirmation above 1 MiB. Full-resolution HTML export remains self-contained. The Memories screen is dreb-only (`~/.dreb/memory` and populated active/on-disk-session project `.dreb/memory`; empty projects are omitted), shows complete indexes with a >200-line warning, opens local index links within the selected scope, replaces stale editor content with visible loading feedback, surfaces malformed entry frontmatter for repair, preserves drafts on conflicts, and does not create/rename entries or expose Claude paths. Compact SSE snapshots update live fleet cards without repeatedly fetching the cross-project inventory, and session drill-in hydrates state, messages, and background agents through one ordered snapshot request. Terminal provider failures show their reason on fleet cards, while transient failures clear terminal state when automatic retry begins and remain recorded inline on the failed attempt. Its top bar and persistent session header indicators report connecting, connected, retrying, resyncing, disconnected, or auth failed; bounded SSE replay plus an explicit snapshot barrier restores session state, tasks, and image references after a reload, restart, gap, backpressure disconnect, or stalled stream, while authenticated image routes recover bytes separately from authoritative transcripts. +**Also available:** [`@dreb/telegram`](https://www.npmjs.com/package/@dreb/telegram) — run dreb as a Telegram bot with live tool status and visible results for user-facing tools (`npm install -g @dreb/telegram`). [`@dreb/dashboard`](https://www.npmjs.com/package/@dreb/dashboard) — run `dreb dashboard` for a browser UI with fleet overview, full chat steering, a TUI-parity rolling TPS indicator with long-term delta, generic fail-closed built-in slash-command discovery and execution, inline provider/API failures with partial output preserved, sanitized raster tool images plus sent user uploads retained as bounded transcript previews by default, a bounded all-agent subagent panel with drill-in, host file browser, dreb memory editor with exact-revision saves and automatic index cleanup on delete, curated appearance themes (per-browser light/dark), and Tailscale/rotating-code pairing (`npm install -g @dreb/dashboard`; see [docs/dashboard.md](docs/dashboard.md)). Tool images cross browser-facing transport as content-addressed references; browser-local Settings offers placeholders, bounded previews, or informed-opt-in originals, with size disclosure and confirmation above 1 MiB. Full-resolution HTML export remains self-contained. The Memories screen is dreb-only (`~/.dreb/memory` and populated active/on-disk-session project `.dreb/memory`; empty projects are omitted), shows complete indexes with a >200-line warning, opens local index links within the selected scope, replaces stale editor content with visible loading feedback, surfaces malformed entry frontmatter for repair, preserves drafts on conflicts, and does not create/rename entries or expose Claude paths. Compact SSE snapshots update live fleet cards without repeatedly fetching the cross-project inventory, and session drill-in hydrates state, messages, and background agents through one ordered snapshot request. Terminal provider failures show their reason on fleet cards, while transient failures clear terminal state when automatic retry begins and remain recorded inline on the failed attempt. Its top bar and persistent session header indicators report connecting, connected, retrying, resyncing, disconnected, or auth failed; bounded SSE replay plus an explicit snapshot barrier restores session state, tasks, and image references after a reload, restart, gap, backpressure disconnect, or stalled stream, while authenticated image routes recover bytes separately from authoritative transcripts. **Platform notes:** [Windows](docs/windows.md) | [Termux (Android)](docs/termux.md) | [tmux](docs/tmux.md) | [Terminal setup](docs/terminal-setup.md) | [Shell aliases](docs/shell-aliases.md) diff --git a/packages/coding-agent/docs/dashboard.md b/packages/coding-agent/docs/dashboard.md index a5c4afae..504f1718 100644 --- a/packages/coding-agent/docs/dashboard.md +++ b/packages/coding-agent/docs/dashboard.md @@ -124,7 +124,7 @@ networking window above. | Screen | What it does | |---|---| | **Fleet** | Home. Live-first: one grid of every live session at the top — status chip (● running / ◆ needs-attention / ○ idle / ✕ error), project path, activity line, live subagent lines, tasks progress, ctx%, model, terminal provider-error reason, last activity. Live cards keep a deterministic order by project path, then session start time; needs-attention cards badge the browser tab without jumping around. Below the grid: past sessions grouped by project, three compact rows per group with an "all N on disk" expander, resume and delete. | -| **Session view** | Full chat drill-in. Markdown streaming transcript (text, thinking blocks with expand preference, inline provider/API failures with partial output preserved, agent-result cards, tool cards with bespoke read/write/edit/bash bodies plus full expandable inputs, markdown-rendered results for markdown-contract tools like subagent/skill/web_fetch/suggest_next, and inline tool-result images, compaction/branch summaries, custom messages), per-message copy, tasks panel, a bounded scrollable subagent panel that lists every retained agent newest-first with full running/done counts, status line with elapsed time plus ■ stop and compaction/retry aborts, a persistent session-header live indicator, and an info bar with cwd, branch, session name, token breakdown, cost/(sub)/daily rollup, ctx%, median tok/s, and a stats popover. Composer supports auto-grow, history, `/` autocomplete from `get_commands`, image attach/paste with sent images retained as user-message previews, queued-message chips with restore-all, steer/follow-up modes, and suggest-next. Registered built-in slash commands are discovered generically, deduplicated ahead of colliding resource commands, and intercepted before prompting: dashboard actions cover settings, model, scoped-models, export/import, name/session stats, fork/tree, new/compact/dream, resume/reload, and quit. `/scoped-models` deep-links to the Settings editor with the session's current cwd as project context; login/logout show an explicit not-yet-implemented notice, while copy/hotkeys/buddy give terminal-only guidance. Future built-ins are intercepted automatically. The RPC prompt boundary rejects any built-in that reaches it during command-loading races or failures, so slash text cannot leak to the model. Attachments are retained and the command is visibly rejected rather than silently discarded. The ⋯ menu covers export HTML, compact, rename, fork-from-message, loaded context, and tool expand/collapse. Session names update live from manual rename or auto-naming. Extension UI requests for select/confirm/input/editor render as modals; a rich `ask`/`ask_user` request renders inline as a single wizard that presents all its questions together — each with Markdown-formatted question text, choices, optional free text — plus an in-card Stop agent action, Escape-to-stop, and the authoritative auto-stop countdown, and is answered as one batch submit. Pending questions set needs-attention state and use the existing hidden-page notification path. Extension notifications render as toasts. | +| **Session view** | Full chat drill-in. Markdown streaming transcript (text, thinking blocks with expand preference, inline provider/API failures with partial output preserved, agent-result cards, tool cards with bespoke read/write/edit/bash bodies plus full expandable inputs, markdown-rendered results for markdown-contract tools like subagent/skill/web_fetch/suggest_next, and inline tool-result images, compaction/branch summaries, custom messages), per-message copy, tasks panel, a bounded scrollable subagent panel that lists every retained agent newest-first with full running/done counts, status line with elapsed time plus ■ stop and compaction/retry aborts, a persistent session-header live indicator, and an info bar with cwd, branch, session name, token breakdown, cost/(sub)/daily rollup, ctx%, a TUI-parity latest-100 median TPS indicator (`~31 tok/s [100] · 10% ↑ median [10000]`), and a stats popover. Composer supports auto-grow, history, `/` autocomplete from `get_commands`, image attach/paste with sent images retained as user-message previews, queued-message chips with restore-all, steer/follow-up modes, and suggest-next. Registered built-in slash commands are discovered generically, deduplicated ahead of colliding resource commands, and intercepted before prompting: dashboard actions cover settings, model, scoped-models, export/import, name/session stats, fork/tree, new/compact/dream, resume/reload, and quit. `/scoped-models` deep-links to the Settings editor with the session's current cwd as project context; login/logout show an explicit not-yet-implemented notice, while copy/hotkeys/buddy give terminal-only guidance. Future built-ins are intercepted automatically. The RPC prompt boundary rejects any built-in that reaches it during command-loading races or failures, so slash text cannot leak to the model. Attachments are retained and the command is visibly rejected rather than silently discarded. The ⋯ menu covers export HTML, compact, rename, fork-from-message, loaded context, and tool expand/collapse. Session names update live from manual rename or auto-naming. Extension UI requests for select/confirm/input/editor render as modals; a rich `ask`/`ask_user` request renders inline as a single wizard that presents all its questions together — each with Markdown-formatted question text, choices, optional free text — plus an in-card Stop agent action, Escape-to-stop, and the authoritative auto-stop countdown, and is answered as one batch submit. Pending questions set needs-attention state and use the existing hidden-page notification path. Extension notifications render as toasts. | | **Subagent view** | Read-only transcript of a background agent: live events via the RPC relay, hydrated from the agent's on-disk session log (`/subagents/:agentId/messages`) so the transcript survives browser reloads. Shows the task, streaming output, tool activity, and any safe Dispatch Arbiter changed/unchanged/failure records with the final agent/model/thinking. No raw arbiter output is displayed or transported. No composer — subagents can't be steered yet; the parent session controls them. | | **Files** | Host-wide browser with places shortcuts (home, /tmp, project roots), breadcrumbs to `/`, new-folder, download, drop-zone/picker upload with explicit collision prompts, and "new session here" on any directory. It also shows the **effective global nested-context trust** for the displayed canonical directory: untrusted, trusted by that root, inherited from a granting root, or global expert trust-all. You can trust the displayed folder and descendants, or untrust the actual granting root; untrusting an inherited folder removes that root's trust for all descendants. | | **Memories** | Dreb-only memory management for `~/.dreb/memory` and `.dreb/memory` under active/disk project roots. It lists existing `MEMORY.md` indexes and direct child `.md` entries only (no Claude paths, create, or rename), shows entry frontmatter or metadata errors so malformed files can be repaired, renders sanitized Markdown preview beside a raw editor, and uses exact SHA-256 revisions so stale saves/deletes return conflicts while preserving drafts. The index view is complete, not truncated, and warns when it exceeds the 200-line memory-index convention. Entry deletion requires both entry and index revisions, removes only matching safe Markdown-link index lines (`file.md` / `./file.md`), writes the cleaned index atomically before unlinking the entry, and rolls back loudly if the unlink fails. | diff --git a/packages/coding-agent/docs/rpc.md b/packages/coding-agent/docs/rpc.md index a848d4c2..8b56fbf6 100644 --- a/packages/coding-agent/docs/rpc.md +++ b/packages/coding-agent/docs/rpc.md @@ -796,16 +796,26 @@ Response: { "provider": "anthropic", "modelId": "claude-sonnet-4", - "median": 31, - "mean": 32, - "count": 100 + "rolling": { + "median": 31, + "mean": 32, + "count": 100 + }, + "delta": { + "baselineMedian": 30, + "recentMedian": 33, + "percentDelta": 10, + "direction": "above", + "baselineCount": 10000, + "recentCount": 10 + } } ] } } ``` -`models` contains per-model rolling averages computed from the agent's performance log. Each entry includes the median TPS, mean TPS, and number of recorded turns for that model. Returns an empty `models` array when no performance data has been recorded. +`models` includes every provider/model identity in the retained performance log. `rolling` comes from the shared latest-100 calculator, while `delta` compares the latest 10 turns with a baseline of up to 10,000 turns. The TUI, dashboard session details, and Telegram `/stats` all use these shared values; they hide TPS until `rolling.count` reaches three and show trend metadata only when both delta counts reach three. With no recorded performance data, `models` is empty. #### export_html diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index ac944418..2bc54ca7 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,6 +1,6 @@ { "name": "@dreb/coding-agent", - "version": "2.55.1", + "version": "2.55.2", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "type": "module", "drebConfig": { diff --git a/packages/coding-agent/src/core/performance-tracker.ts b/packages/coding-agent/src/core/performance-tracker.ts index 17ef0288..1364b7fc 100644 --- a/packages/coding-agent/src/core/performance-tracker.ts +++ b/packages/coding-agent/src/core/performance-tracker.ts @@ -39,6 +39,13 @@ export interface PerformanceDelta { recentCount: number; } +export interface ModelPerformanceSummary { + provider: string; + modelId: string; + rolling: RollingAverage; + delta: PerformanceDelta; +} + export class PerformanceTracker { private static readonly PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000; private static readonly LOCK_TIMEOUT_MS = 1000; @@ -139,33 +146,21 @@ export class PerformanceTracker { }; } - getAllRollingAverages( - windowMs = 24 * 60 * 60 * 1000, - ): Array<{ provider: string; modelId: string; median: number; mean: number; count: number }> { - const cutoff = Date.now() - windowMs; - const filtered = this.entries.filter((e) => entryTime(e) >= cutoff); - - const groups = new Map(); - for (const entry of filtered) { + getAllModelSummaries(): ModelPerformanceSummary[] { + const models = new Map(); + for (const entry of this.entries) { const key = `${entry.provider}\0${entry.modelId}`; - const arr = groups.get(key) ?? []; - arr.push(entry.tps); - groups.set(key, arr); - } - - const results: Array<{ provider: string; modelId: string; median: number; mean: number; count: number }> = []; - for (const [key, values] of groups) { - const [provider, modelId] = key.split("\0"); - results.push({ - provider, - modelId, - median: computeMedian(values), - mean: computeMean(values), - count: values.length, - }); + if (!models.has(key)) { + models.set(key, { provider: entry.provider, modelId: entry.modelId }); + } } - return results; + return Array.from(models.values(), ({ provider, modelId }) => ({ + provider, + modelId, + rolling: this.getRollingAverage(provider, modelId), + delta: this.getPerformanceDelta(provider, modelId), + })); } prune(ageMs = 30 * 24 * 60 * 60 * 1000): void { diff --git a/packages/coding-agent/src/modes/rpc/index.ts b/packages/coding-agent/src/modes/rpc/index.ts index 4c05a688..e77cf7ad 100644 --- a/packages/coding-agent/src/modes/rpc/index.ts +++ b/packages/coding-agent/src/modes/rpc/index.ts @@ -20,6 +20,7 @@ export type { RpcExtensionUIRequest, RpcExtensionUIResponse, RpcPendingMessages, + RpcPerformanceStats, RpcQueuedMessage, RpcResources, RpcResponse, diff --git a/packages/coding-agent/src/modes/rpc/rpc-client.ts b/packages/coding-agent/src/modes/rpc/rpc-client.ts index 518c3659..629b1448 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-client.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-client.ts @@ -23,6 +23,7 @@ import type { RpcExtensionUIRequest, RpcExtensionUIResponse, RpcPendingMessages, + RpcPerformanceStats, RpcResources, RpcResponse, RpcSessionInfo, @@ -579,9 +580,7 @@ export class RpcClient { /** * Get performance statistics. */ - async getPerformanceStats(): Promise<{ - models: Array<{ provider: string; modelId: string; median: number; mean: number; count: number }>; - }> { + async getPerformanceStats(): Promise { const response = await this.send({ type: "get_performance_stats" }); return this.getData(response); } diff --git a/packages/coding-agent/src/modes/rpc/rpc-mode.ts b/packages/coding-agent/src/modes/rpc/rpc-mode.ts index 577666ed..b7184dd1 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-mode.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-mode.ts @@ -69,6 +69,7 @@ import type { RpcExtensionUIRequest, RpcExtensionUIResponse, RpcPendingMessages, + RpcPerformanceStats, RpcResources, RpcResponse, RpcScopedModel, @@ -110,11 +111,9 @@ export function toRpcSessionInfo(s: SessionInfo): RpcSessionInfo { }; } -export function getPerformanceStatsData(session: Pick): { - models: Array<{ provider: string; modelId: string; median: number; mean: number; count: number }>; -} { +export function getPerformanceStatsData(session: Pick): RpcPerformanceStats { const tracker = session.getPerformanceTracker(); - return { models: tracker.getAllRollingAverages() }; + return { models: tracker.getAllModelSummaries() }; } export function getScopedModelsForRpc(session: Pick): RpcScopedModel[] { diff --git a/packages/coding-agent/src/modes/rpc/rpc-types.ts b/packages/coding-agent/src/modes/rpc/rpc-types.ts index ad27e8f7..b81a20bf 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-types.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-types.ts @@ -12,6 +12,7 @@ import type { BashResult } from "../../core/bash-executor.js"; import type { CompactionResult } from "../../core/compaction/index.js"; import type { DispatchArbitrationRecord } from "../../core/dispatch-arbiter.js"; import type { ContextUsage } from "../../core/extensions/types.js"; +import type { ModelPerformanceSummary } from "../../core/performance-tracker.js"; import type { SessionEntry } from "../../core/session-manager.js"; import type { SubagentArbiterSettings } from "../../core/settings-manager.js"; import type { SourceInfo } from "../../core/source-info.js"; @@ -175,6 +176,11 @@ export interface RpcResources { systemPromptPresent: boolean; } +export interface RpcPerformanceStats { + /** TUI-parity summaries computed by the shared rolling and delta calculators. */ + models: ModelPerformanceSummary[]; +} + export interface RpcQueuedMessage { text: string; images?: ImageContent[]; @@ -360,7 +366,7 @@ export type RpcResponse = type: "response"; command: "get_performance_stats"; success: true; - data: { models: Array<{ provider: string; modelId: string; median: number; mean: number; count: number }> }; + data: RpcPerformanceStats; } | { id?: string; type: "response"; command: "export_html"; success: true; data: { path: string } } | { id?: string; type: "response"; command: "import_jsonl"; success: true; data: { cancelled: boolean } } diff --git a/packages/coding-agent/test/performance-tracker.test.ts b/packages/coding-agent/test/performance-tracker.test.ts index f4496e12..ee240b65 100644 --- a/packages/coding-agent/test/performance-tracker.test.ts +++ b/packages/coding-agent/test/performance-tracker.test.ts @@ -184,46 +184,76 @@ describe("PerformanceTracker", () => { expect(delta.recentCount).toBe(2); }); - // getAllRollingAverages() -------------------------------------------------- + // getAllModelSummaries() --------------------------------------------------- - it("getAllRollingAverages() returns all provider/model combinations", () => { + it("getAllModelSummaries() returns an empty array when no data is recorded", () => { tracker = new PerformanceTracker(logPath); - tracker.record(makeEntry({ provider: "anthropic", modelId: "claude-3-sonnet", tps: 10 })); - tracker.record(makeEntry({ provider: "anthropic", modelId: "claude-3-sonnet", tps: 20 })); - tracker.record(makeEntry({ provider: "anthropic", modelId: "claude-3-opus", tps: 30 })); - tracker.record(makeEntry({ provider: "openai", modelId: "gpt-4", tps: 40 })); - - const results = tracker.getAllRollingAverages(); - expect(results).toHaveLength(3); - - const sonnet = results.find((r) => r.provider === "anthropic" && r.modelId === "claude-3-sonnet"); - expect(sonnet).toBeDefined(); - expect(sonnet!.count).toBe(2); - expect(sonnet!.mean).toBe(15); - expect(sonnet!.median).toBe(15); - - const opus = results.find((r) => r.provider === "anthropic" && r.modelId === "claude-3-opus"); - expect(opus).toBeDefined(); - expect(opus!.count).toBe(1); - expect(opus!.mean).toBe(30); - - const gpt4 = results.find((r) => r.provider === "openai" && r.modelId === "gpt-4"); - expect(gpt4).toBeDefined(); - expect(gpt4!.count).toBe(1); - expect(gpt4!.mean).toBe(40); + expect(tracker.getAllModelSummaries()).toEqual([]); }); - it("getAllRollingAverages() respects the window", () => { + it("getAllModelSummaries() includes retained identities and delegates to the shared calculators", () => { tracker = new PerformanceTracker(logPath); - tracker.record( - makeEntryAtOffsetMs(25 * 60 * 60 * 1000, { provider: "anthropic", modelId: "claude-3-sonnet", tps: 999 }), + for (let i = 0; i < 3; i++) { + tracker.record( + makeEntryAtOffsetMs((25 * 60 * 60 + i) * 1000, { + provider: "anthropic", + modelId: "claude-3-sonnet", + tps: 30, + }), + ); + tracker.record(makeEntryAtOffsetMs(i * 1000, { provider: "openai", modelId: "gpt-4", tps: 40 })); + } + const rollingSpy = vi.spyOn(tracker, "getRollingAverage"); + const deltaSpy = vi.spyOn(tracker, "getPerformanceDelta"); + + const summaries = tracker.getAllModelSummaries(); + + expect(summaries).toHaveLength(2); + expect(summaries.find((summary) => summary.provider === "anthropic")).toMatchObject({ + modelId: "claude-3-sonnet", + rolling: { median: 30, mean: 30, count: 3 }, + delta: { baselineMedian: 30, recentMedian: 30, direction: "stable", baselineCount: 3, recentCount: 3 }, + }); + expect(rollingSpy).toHaveBeenCalledWith("anthropic", "claude-3-sonnet"); + expect(rollingSpy).toHaveBeenCalledWith("openai", "gpt-4"); + expect(deltaSpy).toHaveBeenCalledWith("anthropic", "claude-3-sonnet"); + expect(deltaSpy).toHaveBeenCalledWith("openai", "gpt-4"); + }); + + it("getAllModelSummaries() uses the newest 100 samples for its rolling statistics", () => { + tracker = new PerformanceTracker(logPath); + for (let i = 0; i < 102; i++) { + tracker.record(makeEntryAtOffsetMs((102 - i) * 1000, { tps: i + 1 })); + } + + const [summary] = tracker.getAllModelSummaries(); + expect(summary.rolling).toEqual({ median: (3 + 102) / 2, mean: (3 + 102) / 2, count: 100 }); + expect(summary.delta.baselineCount).toBe(102); + expect(summary.delta.recentCount).toBe(10); + }); + + it("getAllModelSummaries() caps its delta baseline at the newest 10,000 samples", () => { + const entryCount = 10_002; + const now = Date.now(); + const lines = Array.from({ length: entryCount }, (_, index) => + JSON.stringify( + makeEntry({ + timestamp: new Date(now - (entryCount - index) * 1000).toISOString(), + tps: index + 1, + }), + ), ); - tracker.record(makeEntryAtOffsetMs(60 * 60 * 1000, { provider: "openai", modelId: "gpt-4", tps: 40 })); + writeFileSync(logPath, `${lines.join("\n")}\n`, "utf8"); + tracker = new PerformanceTracker(logPath); + + const [summary] = tracker.getAllModelSummaries(); - const results = tracker.getAllRollingAverages(); - expect(results).toHaveLength(1); - expect(results[0].provider).toBe("openai"); - expect(results[0].modelId).toBe("gpt-4"); + expect(summary.delta).toMatchObject({ + baselineCount: 10_000, + recentCount: 10, + baselineMedian: (3 + 10_002) / 2, + recentMedian: (9_993 + 10_002) / 2, + }); }); // prune() ------------------------------------------------------------------ diff --git a/packages/coding-agent/test/rpc-performance.test.ts b/packages/coding-agent/test/rpc-performance.test.ts index 3697ec62..4f3a2c15 100644 --- a/packages/coding-agent/test/rpc-performance.test.ts +++ b/packages/coding-agent/test/rpc-performance.test.ts @@ -3,22 +3,33 @@ import { RpcClient } from "../src/modes/rpc/rpc-client.js"; import { getPerformanceStatsData } from "../src/modes/rpc/rpc-mode.js"; describe("RPC performance stats", () => { - it("builds performance stats response data from the session tracker", () => { - const models = [{ provider: "anthropic", modelId: "claude-3-sonnet", median: 30, mean: 31, count: 4 }]; - const session = { - getPerformanceTracker: () => ({ - getAllRollingAverages: vi.fn(() => models), - }), - }; + const models = [ + { + provider: "anthropic", + modelId: "claude-3-sonnet", + rolling: { median: 32, mean: 33, count: 100 }, + delta: { + baselineMedian: 30, + recentMedian: 36, + percentDelta: 20, + direction: "above" as const, + baselineCount: 200, + recentCount: 10, + }, + }, + ]; + + it("builds shared model summaries from the session tracker", () => { + const tracker = { getAllModelSummaries: vi.fn(() => models) }; + const session = { getPerformanceTracker: () => tracker }; expect(getPerformanceStatsData(session as any)).toEqual({ models }); + expect(tracker.getAllModelSummaries).toHaveBeenCalledOnce(); }); it("RpcClient.getPerformanceStats sends the get_performance_stats command", async () => { const client = new RpcClient() as any; - const data = { - models: [{ provider: "anthropic", modelId: "claude-3-sonnet", median: 30, mean: 31, count: 4 }], - }; + const data = { models }; client.send = vi.fn().mockResolvedValue({ type: "response", command: "get_performance_stats", diff --git a/packages/dashboard/README.md b/packages/dashboard/README.md index ad3e3ba5..41c5d356 100644 --- a/packages/dashboard/README.md +++ b/packages/dashboard/README.md @@ -47,7 +47,7 @@ Open `http://127.0.0.1:5343`. settings/model/scoped-models, import/export, session tree, fork, new/compact/dream, resume/reload, and quit), image attach/paste with sent-image previews retained in user transcript entries, queued-message restore, persistent session-header live indicator, footer-parity info bar (branch, tokens, cost, ctx%, - median tok/s), stats/loaded-context/fork modals, steer/follow-up composer + latest-100 median tok/s with sample count and long-term delta), stats/loaded-context/fork modals, steer/follow-up composer modes, ■ abort, model/thinking switchers, extension-UI modals, export HTML, and live auto-naming. - **Subagent drill-in** — read-only transcript of a background agent: live diff --git a/packages/dashboard/package.json b/packages/dashboard/package.json index 9f5c9e72..9671a160 100644 --- a/packages/dashboard/package.json +++ b/packages/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "@dreb/dashboard", - "version": "2.55.1", + "version": "2.55.2", "description": "Web dashboard for dreb — fleet overview, chat parity, subagent observability", "license": "MIT", "type": "module", diff --git a/packages/dashboard/src/client/screens/session.tsx b/packages/dashboard/src/client/screens/session.tsx index 853c704f..9d3e2f6c 100644 --- a/packages/dashboard/src/client/screens/session.tsx +++ b/packages/dashboard/src/client/screens/session.tsx @@ -10,6 +10,7 @@ import type { ImageAttachmentDto, ModelInfoDto, PendingMessagesDto, + PerformanceModelSummaryDto, PerformanceStatsDto, QueuedMessageDto, ResourcesDto, @@ -75,6 +76,28 @@ export function formatTokens(count: number): string { return `${Math.round(count / 1000000)}M`; } +export function formatPerformanceIndicator(summary: PerformanceModelSummaryDto | undefined): string | undefined { + if (!summary || summary.rolling.count < 3) return undefined; + + const arrows = { above: "↑", below: "↓", stable: "→" } as const; + const deltaPercent = summary.delta.direction === "stable" ? 0 : Math.round(Math.abs(summary.delta.percentDelta)); + const medianDelta = + summary.delta.recentCount >= 3 && summary.delta.baselineCount >= 3 + ? ` · ${deltaPercent}% ${arrows[summary.delta.direction]} median [${summary.delta.baselineCount}]` + : ""; + + return `~${Math.round(summary.rolling.median)} tok/s [${summary.rolling.count}]${medianDelta}`; +} + +export function performanceIndicatorForModel( + performance: PerformanceStatsDto | undefined, + model: Pick | undefined, +): string | undefined { + if (!model) return undefined; + const summary = performance?.models.find((entry) => entry.provider === model.provider && entry.modelId === model.id); + return formatPerformanceIndicator(summary); +} + function shortenPath(path: string): string { return path.replace(/^\/home\/[^/]+/, "~"); } @@ -1634,15 +1657,7 @@ export function SessionScreen(props: { store: AppStore; sessionKey: string }): J const percent = usage.percent === null ? "?" : `${usage.percent.toFixed(0)}%`; return `ctx ${percent}/${formatTokens(usage.contextWindow)}`; }; - const tokPerSecond = () => { - const model = runtime()?.state.model; - if (!model) return undefined; - const rolling = performance()?.models.find( - (entry) => entry.provider === model.provider && entry.modelId === model.id, - ); - if (!rolling || rolling.count < 3) return undefined; - return `${Math.round(rolling.median)} tok/s`; - }; + const tokPerSecond = () => performanceIndicatorForModel(performance(), runtime()?.state.model); const infoStats = () => [tokenSummary(), costSummary(), contextSummary(), tokPerSecond()].filter(Boolean) as string[]; const pendingMessageItems = () => [ diff --git a/packages/dashboard/src/shared/protocol.ts b/packages/dashboard/src/shared/protocol.ts index b1681a8a..96ba8814 100644 --- a/packages/dashboard/src/shared/protocol.ts +++ b/packages/dashboard/src/shared/protocol.ts @@ -138,8 +138,22 @@ export interface SessionStatsDto { contextUsage?: ContextUsageDto; } +export interface PerformanceModelSummaryDto { + provider: string; + modelId: string; + rolling: { median: number; mean: number; count: number }; + delta: { + baselineMedian: number; + recentMedian: number; + percentDelta: number; + direction: "above" | "below" | "stable"; + baselineCount: number; + recentCount: number; + }; +} + export interface PerformanceStatsDto { - models: Array<{ provider: string; modelId: string; median: number; mean: number; count: number }>; + models: PerformanceModelSummaryDto[]; } export interface ResourcesDto { diff --git a/packages/dashboard/test/client/fleet-mobile.browser.test.ts b/packages/dashboard/test/client/fleet-mobile.browser.test.ts index 408f5850..34d52aa0 100644 --- a/packages/dashboard/test/client/fleet-mobile.browser.test.ts +++ b/packages/dashboard/test/client/fleet-mobile.browser.test.ts @@ -86,7 +86,7 @@ function makeFakeRuntimeClient(): FakeRuntimeClient { }), getLastAssistantText: async () => undefined, listBackgroundAgents: async () => [], - getPerformanceStats: async () => ({}), + getPerformanceStats: async () => ({ models: [] }), getGitBranch: async () => null, getDailyCost: async () => 0, getCommands: async () => [], diff --git a/packages/dashboard/test/client/screens.test.tsx b/packages/dashboard/test/client/screens.test.tsx index cfaecc92..364dee03 100644 --- a/packages/dashboard/test/client/screens.test.tsx +++ b/packages/dashboard/test/client/screens.test.tsx @@ -260,7 +260,12 @@ import { FilesScreen } from "../../src/client/screens/files.js"; import { FleetScreen, fleetGroupKey } from "../../src/client/screens/fleet.js"; import { MemoriesScreen } from "../../src/client/screens/memories.js"; import { PairingScreen } from "../../src/client/screens/pairing.js"; -import { formatTokens, SessionScreen } from "../../src/client/screens/session.js"; +import { + formatPerformanceIndicator, + formatTokens, + performanceIndicatorForModel, + SessionScreen, +} from "../../src/client/screens/session.js"; import { SettingsScreen } from "../../src/client/screens/settings.js"; import { SubagentScreen } from "../../src/client/screens/subagent.js"; import { @@ -282,12 +287,38 @@ import { createAppStore } from "../../src/client/state/store.js"; import { type CommandDto, MAX_TOTAL_IMAGE_BYTES, + type PerformanceModelSummaryDto, + type PerformanceStatsDto, type RuntimeInfoDto, type SettingsDto, } from "../../src/shared/protocol.js"; const disposers: Array<() => void> = []; +function performanceSummary( + overrides: { + provider?: string; + modelId?: string; + rolling?: Partial; + delta?: Partial; + } = {}, +): PerformanceModelSummaryDto { + return { + provider: overrides.provider ?? "test", + modelId: overrides.modelId ?? "test-model", + rolling: { median: 41.8, mean: 43, count: 100, ...overrides.rolling }, + delta: { + baselineMedian: 38, + recentMedian: 41.8, + percentDelta: 10, + direction: "above", + baselineCount: 200, + recentCount: 10, + ...overrides.delta, + }, + }; +} + // jsdom lacks ResizeObserver; real browsers always have it. Install a no-op so // the stick-to-bottom controller's observeContent() attaches quietly instead of // logging its (correct) "ResizeObserver unavailable" warning on every screen @@ -1120,7 +1151,10 @@ describe("screen smoke tests", () => { vi.mocked(api.branch).mockResolvedValue({ branch: "feature/info" }); vi.mocked(api.dailyCost).mockResolvedValue({ cost: 1.25 }); vi.mocked(api.performance).mockResolvedValue({ - models: [{ provider: "test", modelId: "test-model", median: 41.8, mean: 43, count: 4 }], + models: [ + performanceSummary({ rolling: { count: 4 }, delta: { baselineCount: 4, recentCount: 4 } }), + performanceSummary({ modelId: "other-model", rolling: { median: 99 } }), + ], }); const store = makeStore() as any; // Inject session state directly (store internals sync from the reducer). @@ -1171,7 +1205,8 @@ describe("screen smoke tests", () => { expect(el.textContent).toContain("~/software/dreb (feature/info) • test session"); expect(el.textContent).toContain("↑1.2k ↓45k W12"); expect(el.textContent).toContain("$0.420 (sub), today: $1.25"); - expect(el.textContent).toContain("42 tok/s"); + expect(el.textContent).toContain("~42 tok/s [4] · 10% ↑ median [4]"); + expect(el.textContent).not.toContain("99 tok/s"); expect(el.textContent).toContain("test/test-model"); expect(el.textContent).toContain("scan things"); // Suggest-next chip @@ -3527,6 +3562,33 @@ describe("dashboard client regressions", () => { expect(formatTokens(12_000_000)).toBe("12M"); }); + it("formats above, below, and stable TPS summaries like the TUI footer", () => { + expect(formatPerformanceIndicator(performanceSummary())).toBe("~42 tok/s [100] · 10% ↑ median [200]"); + expect( + formatPerformanceIndicator(performanceSummary({ delta: { direction: "below", percentDelta: -9.6 } })), + ).toBe("~42 tok/s [100] · 10% ↓ median [200]"); + expect(formatPerformanceIndicator(performanceSummary({ delta: { direction: "stable", percentDelta: 99 } }))).toBe( + "~42 tok/s [100] · 0% → median [200]", + ); + }); + + it("applies the TUI sample gates to TPS summaries", () => { + expect(formatPerformanceIndicator(performanceSummary({ rolling: { count: 2 } }))).toBeUndefined(); + expect(formatPerformanceIndicator(undefined)).toBeUndefined(); + expect(formatPerformanceIndicator(performanceSummary({ delta: { recentCount: 2 } }))).toBe("~42 tok/s [100]"); + expect(formatPerformanceIndicator(performanceSummary({ delta: { baselineCount: 2 } }))).toBe("~42 tok/s [100]"); + }); + + it("selects only the active model's shared TPS summary", () => { + const performance: PerformanceStatsDto = { models: [performanceSummary()] }; + expect(performanceIndicatorForModel(performance, { provider: "test", id: "test-model" })).toBe( + "~42 tok/s [100] · 10% ↑ median [200]", + ); + expect(performanceIndicatorForModel(performance, { provider: "test", id: "other-model" })).toBeUndefined(); + expect(performanceIndicatorForModel(undefined, { provider: "test", id: "test-model" })).toBeUndefined(); + expect(performanceIndicatorForModel(performance, undefined)).toBeUndefined(); + }); + it("transcript render item wrappers stay stable for unchanged rows", () => { const entries: Parameters[0] = [ { kind: "user", text: "first prompt" }, diff --git a/packages/dashboard/test/runtime-pool.test.ts b/packages/dashboard/test/runtime-pool.test.ts index aad2370c..20bb4b90 100644 --- a/packages/dashboard/test/runtime-pool.test.ts +++ b/packages/dashboard/test/runtime-pool.test.ts @@ -68,7 +68,21 @@ export function makeFakeClient() { cost: 0.42, })), getPerformanceStats: vi.fn(async () => ({ - models: [{ provider: "test", modelId: "m1", median: 42, mean: 43, count: 4 }], + models: [ + { + provider: "test", + modelId: "m1", + rolling: { median: 41, mean: 42, count: 4 }, + delta: { + baselineMedian: 41, + recentMedian: 41, + percentDelta: 0, + direction: "stable" as const, + baselineCount: 4, + recentCount: 4, + }, + }, + ], })), getResources: vi.fn(async () => ({ contextFiles: [{ path: "/tmp/AGENTS.md" }], diff --git a/packages/dashboard/test/server.test.ts b/packages/dashboard/test/server.test.ts index ee71267b..b0d81550 100644 --- a/packages/dashboard/test/server.test.ts +++ b/packages/dashboard/test/server.test.ts @@ -732,7 +732,21 @@ describe("dashboard server — fleet and runtimes", () => { const { key } = (await create.json()) as { key: string }; await expect(fetch(`${base}/api/runtimes/${key}/performance`).then((r) => r.json())).resolves.toEqual({ - models: [{ provider: "test", modelId: "m1", median: 42, mean: 43, count: 4 }], + models: [ + { + provider: "test", + modelId: "m1", + rolling: { median: 41, mean: 42, count: 4 }, + delta: { + baselineMedian: 41, + recentMedian: 41, + percentDelta: 0, + direction: "stable", + baselineCount: 4, + recentCount: 4, + }, + }, + ], }); await expect(fetch(`${base}/api/runtimes/${key}/resources`).then((r) => r.json())).resolves.toEqual({ contextFiles: [{ path: "/tmp/AGENTS.md" }], diff --git a/packages/semantic-search/.claude-plugin/plugin.json b/packages/semantic-search/.claude-plugin/plugin.json index 6edfdc89..20b9c713 100644 --- a/packages/semantic-search/.claude-plugin/plugin.json +++ b/packages/semantic-search/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "semantic-search", "description": "Semantic codebase search — natural language queries over code and docs using embeddings, tree-sitter parsing, and POEM multi-signal ranking", - "version": "2.55.1", + "version": "2.55.2", "author": { "name": "Drew Brereton" }, diff --git a/packages/semantic-search/package.json b/packages/semantic-search/package.json index 1c9e698a..c803455e 100644 --- a/packages/semantic-search/package.json +++ b/packages/semantic-search/package.json @@ -1,6 +1,6 @@ { "name": "@dreb/semantic-search", - "version": "2.55.1", + "version": "2.55.2", "description": "Semantic codebase search engine with embedding-based ranking and MCP server", "publishConfig": { "access": "public" diff --git a/packages/telegram/README.md b/packages/telegram/README.md index 95342f17..a92a5e6c 100644 --- a/packages/telegram/README.md +++ b/packages/telegram/README.md @@ -85,7 +85,7 @@ systemctl --user enable --now dreb-telegram ### Agent - `/status` — Connection & version info -- `/stats` — Token usage, cost, and per-model performance stats (rolling TPS) +- `/stats` — Token usage, cost, and per-model latest-100 median TPS with long-term delta (matching the TUI and dashboard) - `/compact` — Compact context - `/model [pattern]` — View/switch model - `/thinking [level]` — View/set thinking level diff --git a/packages/telegram/package.json b/packages/telegram/package.json index 6f1171c5..0ccbcc7e 100644 --- a/packages/telegram/package.json +++ b/packages/telegram/package.json @@ -1,6 +1,6 @@ { "name": "@dreb/telegram", - "version": "2.55.1", + "version": "2.55.2", "description": "Telegram bot frontend for dreb coding agent", "license": "MIT", "type": "module", diff --git a/packages/telegram/src/agent-bridge.ts b/packages/telegram/src/agent-bridge.ts index c292efce..b0c59f82 100644 --- a/packages/telegram/src/agent-bridge.ts +++ b/packages/telegram/src/agent-bridge.ts @@ -5,7 +5,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { RpcClient, type RpcSessionInfo } from "@dreb/coding-agent/rpc"; +import { RpcClient, type RpcPerformanceStats, type RpcSessionInfo } from "@dreb/coding-agent/rpc"; import type { Config } from "./config.js"; import { log } from "./util/telegram.js"; @@ -239,7 +239,7 @@ export class AgentBridge { /** * Get performance statistics. */ - async getPerformanceStats(): Promise { + async getPerformanceStats(): Promise { if (!this.client) return null; try { return await this.client.getPerformanceStats(); diff --git a/packages/telegram/src/commands/agent.ts b/packages/telegram/src/commands/agent.ts index 4a5c3ff5..67fbd74d 100644 --- a/packages/telegram/src/commands/agent.ts +++ b/packages/telegram/src/commands/agent.ts @@ -89,10 +89,20 @@ export async function cmdStats(ctx: Context, userState: UserState): Promise 0) { - lines.push("\n⚡ *Performance (last 24h):*"); - for (const m of perf.models) { - lines.push(` ${m.provider}/${m.modelId}: ~${m.median.toFixed(1)} tok/s (n=${m.count})`); + const models = (perf?.models ?? []).filter((model) => model.rolling.count >= 3); + if (models.length > 0) { + lines.push("\n⚡ *Performance:*"); + const arrows = { above: "↑", below: "↓", stable: "→" } as const; + for (const model of models) { + const deltaPercent = + model.delta.direction === "stable" ? 0 : Math.round(Math.abs(model.delta.percentDelta)); + const medianDelta = + model.delta.recentCount >= 3 && model.delta.baselineCount >= 3 + ? ` · ${deltaPercent}% ${arrows[model.delta.direction]} median [${model.delta.baselineCount}]` + : ""; + lines.push( + ` ${model.provider}/${model.modelId}: ~${Math.round(model.rolling.median)} tok/s [${model.rolling.count}]${medianDelta}`, + ); } } } catch (e) { diff --git a/packages/telegram/test/commands.test.ts b/packages/telegram/test/commands.test.ts index 4399d6a1..2a4ee26d 100644 --- a/packages/telegram/test/commands.test.ts +++ b/packages/telegram/test/commands.test.ts @@ -1,4 +1,5 @@ import { join } from "node:path"; +import type { RpcPerformanceStats } from "@dreb/coding-agent/rpc"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { Config } from "../src/config.js"; import type { UserState } from "../src/types.js"; @@ -328,9 +329,7 @@ describe("cmdStats", () => { cost: number; contextUsage: { percent: number; tokens: number; contextWindow: number }; }>; - perf?: { - models: Array<{ provider: string; modelId: string; median: number; mean: number; count: number }>; - } | null; + perf?: RpcPerformanceStats | null; }) { return { isAlive: true, @@ -346,7 +345,21 @@ describe("cmdStats", () => { ), getPerformanceStats: vi.fn().mockResolvedValue( overrides?.perf ?? { - models: [{ provider: "anthropic", modelId: "claude-3-sonnet", median: 30.5, mean: 32, count: 100 }], + models: [ + { + provider: "anthropic", + modelId: "claude-3-sonnet", + rolling: { median: 30.5, mean: 32, count: 100 }, + delta: { + baselineMedian: 25, + recentMedian: 30, + percentDelta: 20, + direction: "above" as const, + baselineCount: 10000, + recentCount: 10, + }, + }, + ], }, ), } as any; @@ -357,8 +370,43 @@ describe("cmdStats", () => { await cmdStats(ctx, userState); const sentMessage = mockSafeSend.mock.calls[0][2] as string; - expect(sentMessage).toContain("⚡ *Performance (last 24h):*"); - expect(sentMessage).toContain("anthropic/claude-3-sonnet: ~30.5 tok/s (n=100)"); + expect(sentMessage).toContain("⚡ *Performance:*"); + expect(sentMessage).toContain("anthropic/claude-3-sonnet: ~31 tok/s [100] · 20% ↑ median [10000]"); + expect(sentMessage).not.toContain("last 24h"); + }); + + it.each([ + { + name: "below trend", + delta: { direction: "below" as const, percentDelta: -19.6, recentCount: 10, baselineCount: 10000 }, + expected: " anthropic/claude-3-sonnet: ~31 tok/s [100] · 20% ↓ median [10000]", + }, + { + name: "stable trend", + delta: { direction: "stable" as const, percentDelta: 0.8, recentCount: 10, baselineCount: 10000 }, + expected: " anthropic/claude-3-sonnet: ~31 tok/s [100] · 0% → median [10000]", + }, + { + name: "insufficient recent samples", + delta: { direction: "above" as const, percentDelta: 20, recentCount: 2, baselineCount: 10000 }, + expected: " anthropic/claude-3-sonnet: ~31 tok/s [100]", + }, + { + name: "insufficient baseline samples", + delta: { direction: "above" as const, percentDelta: 20, recentCount: 10, baselineCount: 2 }, + expected: " anthropic/claude-3-sonnet: ~31 tok/s [100]", + }, + ])("formats $name with TUI semantics", async ({ delta, expected }) => { + const bridge = createMockBridge(); + const perf = await bridge.getPerformanceStats(); + Object.assign(perf.models[0].delta, delta); + const userState = createUserState({ bridge }); + + await cmdStats(ctx, userState); + + const sentMessage = mockSafeSend.mock.calls[0][2] as string; + const modelLine = sentMessage.split("\n").find((line) => line.includes("anthropic/claude-3-sonnet")); + expect(modelLine).toBe(expected); }); it("omits performance section when models array is empty", async () => { @@ -366,7 +414,18 @@ describe("cmdStats", () => { await cmdStats(ctx, userState); const sentMessage = mockSafeSend.mock.calls[0][2] as string; - expect(sentMessage).not.toContain("⚡ *Performance (last 24h):*"); + expect(sentMessage).not.toContain("⚡ *Performance:*"); + }); + + it("omits models with fewer than three rolling samples", async () => { + const bridge = createMockBridge(); + const perf = await bridge.getPerformanceStats(); + perf.models[0].rolling.count = 2; + const userState = createUserState({ bridge }); + await cmdStats(ctx, userState); + + const sentMessage = mockSafeSend.mock.calls[0][2] as string; + expect(sentMessage).not.toContain("⚡ *Performance:*"); }); it("omits performance section when getPerformanceStats throws", async () => { @@ -376,7 +435,7 @@ describe("cmdStats", () => { await cmdStats(ctx, userState); const sentMessage = mockSafeSend.mock.calls[0][2] as string; - expect(sentMessage).not.toContain("⚡ *Performance (last 24h):*"); + expect(sentMessage).not.toContain("⚡ *Performance:*"); expect(sentMessage).toContain("Session Stats"); }); diff --git a/packages/tui/package.json b/packages/tui/package.json index 659510e0..6ba5e7b4 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,6 +1,6 @@ { "name": "@dreb/tui", - "version": "2.55.1", + "version": "2.55.2", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "type": "module", "main": "dist/index.js",