diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a72ca0f..388d3be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: pnpm - name: Install diff --git a/openspec/changes/P4-execution-signals/proposal.md b/openspec/changes/P4-execution-signals/proposal.md new file mode 100644 index 0000000..92e6064 --- /dev/null +++ b/openspec/changes/P4-execution-signals/proposal.md @@ -0,0 +1,16 @@ +# P4 — Execution Signals Extractor + +## Why +To implement intelligent, adaptive routing, the router must understand the status and complexity of the developer session. Modlane does this by mining execution signals (consecutive errors, files touched, repeated edits, context size) from incoming request history. + +## What changes +- `src/providers/types.ts`: added `dialect` and `rawBody` properties to neutral `ChatRequest`. +- `src/server.ts`: populated `dialect` and `rawBody` on request parsing. +- `src/signals.ts` [NEW]: implements the `extractSignals` parser, extracting tool usage, results, test failures, files modified, consecutive failures, and repeated edits from the raw dialect messages. +- `src/signals.test.ts` [NEW]: unit tests covering signal extraction for both OpenAI and Anthropic formats. + +## Impact +Allows downstream classification (P5) and routing (P6) systems to inspect the current state of a development session and make routing decisions based on actual execution metrics rather than stubs. + +## Status +Done — implemented, fully tested, and compiling. diff --git a/openspec/changes/P4-execution-signals/tasks.md b/openspec/changes/P4-execution-signals/tasks.md new file mode 100644 index 0000000..a24d62e --- /dev/null +++ b/openspec/changes/P4-execution-signals/tasks.md @@ -0,0 +1,8 @@ +# P4 — Tasks + +- [x] Extend `ChatRequest` in `src/providers/types.ts` to retain `dialect` and `rawBody`. +- [x] Update request parser in `src/server.ts` to populate `dialect` and `rawBody`. +- [x] Implement signal extraction and heuristics in `src/signals.ts`. +- [x] Cover extraction rules for OpenAI format with unit tests in `src/signals.test.ts`. +- [x] Cover extraction rules for Anthropic format with unit tests in `src/signals.test.ts`. +- [x] Verify build and vitest run successfully. diff --git a/src/providers/types.ts b/src/providers/types.ts index 3f7035f..1d39079 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -17,8 +17,12 @@ export interface ChatRequest { stream?: boolean; /** Provider-native tool definitions, passed through unchanged. */ tools?: unknown; + /** The inbound dialect and original raw body for execution signal mining. */ + dialect?: "openai" | "anthropic"; + rawBody?: unknown; } + export interface Usage { promptTokens: number | null; completionTokens: number | null; diff --git a/src/server.ts b/src/server.ts index 9367d63..3ba2beb 100644 --- a/src/server.ts +++ b/src/server.ts @@ -62,6 +62,8 @@ async function handleChat(config: Config, req: IncomingMessage, res: ServerRespo } const chatReq = dialect === "openai" ? parseOpenAIRequest(body) : parseAnthropicRequest(body); + chatReq.dialect = dialect; + chatReq.rawBody = body; const requestedModel = typeof body.model === "string" ? body.model : ""; if (chatReq.stream) { diff --git a/src/signals.test.ts b/src/signals.test.ts new file mode 100644 index 0000000..a5604ff --- /dev/null +++ b/src/signals.test.ts @@ -0,0 +1,223 @@ +import { expect, test } from "vitest"; +import type { ChatRequest } from "./providers/types.js"; +import { extractSignals } from "./signals.js"; + +test("extracts basic metrics from ChatRequest", () => { + const req: ChatRequest = { + model: "test-model", + messages: [ + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" }, + ], + }; + + const signals = extractSignals(req); + expect(signals.totalMessages).toBe(2); + expect(signals.totalCharacters).toBe(14); // 5 + 9 + expect(signals.toolCalls).toHaveLength(0); + expect(signals.toolResults).toHaveLength(0); + expect(signals.filesTouched).toHaveLength(0); + expect(signals.repeatedEdits).toBe(false); + expect(signals.hasTestFailures).toBe(false); + expect(signals.consecutiveFailures).toBe(0); +}); + +test("extracts OpenAI tool calls and results", () => { + const rawBody = { + messages: [ + { role: "user", content: "fix tests" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "edit_file", + arguments: JSON.stringify({ path: "src/server.ts", content: "new code" }), + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "call_1", + content: "Success", + }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_2", + type: "function", + function: { + name: "run_command", + arguments: JSON.stringify({ command: "pnpm test" }), + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "call_2", + content: "FAIL: test_server failed with code 1", + }, + ], + }; + + const req: ChatRequest = { + model: "test-model", + messages: [ + { role: "user", content: "fix tests" }, + { role: "assistant", content: "" }, + { role: "tool", content: "Success" }, + { role: "assistant", content: "" }, + { role: "tool", content: "FAIL: test_server failed with code 1" }, + ], + dialect: "openai", + rawBody, + }; + + const signals = extractSignals(req); + expect(signals.toolCalls).toHaveLength(2); + expect(signals.toolCalls[0].name).toBe("edit_file"); + expect(signals.toolResults).toHaveLength(2); + expect(signals.toolResults[1].isError).toBe(true); + expect(signals.filesTouched).toEqual(["src/server.ts"]); + expect(signals.hasTestFailures).toBe(true); + expect(signals.consecutiveFailures).toBe(1); // last one failed, previous succeeded +}); + +test("extracts Anthropic tool calls and results", () => { + const rawBody = { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call_a1", + name: "str_replace_editor", + input: { path: "src/server.ts", replacement: "foo" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call_a1", + content: "Success replacing text", + is_error: false, + }, + ], + }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call_a2", + name: "str_replace_editor", + input: { path: "src/server.ts", replacement: "bar" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call_a2", + content: "Some compilation warning", + is_error: true, + }, + ], + }, + ], + }; + + const req: ChatRequest = { + model: "test-model", + messages: [ + { role: "assistant", content: "" }, + { role: "user", content: "Success replacing text" }, + { role: "assistant", content: "" }, + { role: "user", content: "Some compilation warning" }, + ], + dialect: "anthropic", + rawBody, + }; + + const signals = extractSignals(req); + expect(signals.toolCalls).toHaveLength(2); + expect(signals.toolResults).toHaveLength(2); + expect(signals.toolResults[1].isError).toBe(true); + expect(signals.filesTouched).toEqual(["src/server.ts"]); + expect(signals.repeatedEdits).toBe(true); // src/server.ts was edited twice in a row + expect(signals.consecutiveFailures).toBe(1); +}); + +test("correctly calculates consecutive failures", () => { + const rawBody = { + messages: [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "1", name: "tool1", input: {} }, + { type: "tool_use", id: "2", name: "tool2", input: {} }, + { type: "tool_use", id: "3", name: "tool3", input: {} }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "1", content: "err1", is_error: true }, + { type: "tool_result", tool_use_id: "2", content: "err2", is_error: true }, + { type: "tool_result", tool_use_id: "3", content: "err3", is_error: true }, + ], + }, + ], + }; + + const req: ChatRequest = { + model: "test-model", + messages: [ + { role: "assistant", content: "" }, + { role: "user", content: "" }, + ], + dialect: "anthropic", + rawBody, + }; + + const signals = extractSignals(req); + expect(signals.consecutiveFailures).toBe(3); +}); + +test("does not flag benign words containing 'fail' as failures", () => { + const rawBody = { + messages: [ + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "1", content: "failover completed; failsafe engaged" }, + ], + }, + ], + }; + + const req: ChatRequest = { + model: "test-model", + messages: [{ role: "user", content: "" }], + dialect: "anthropic", + rawBody, + }; + + const signals = extractSignals(req); + expect(signals.toolResults[0].isError).toBe(false); + expect(signals.hasTestFailures).toBe(false); + expect(signals.consecutiveFailures).toBe(0); +}); diff --git a/src/signals.ts b/src/signals.ts new file mode 100644 index 0000000..49a07b1 --- /dev/null +++ b/src/signals.ts @@ -0,0 +1,184 @@ +import type { ChatRequest } from "./providers/types.js"; + +export interface ToolCallSignal { + name: string; + arguments?: string | Record; +} + +export interface ToolResultSignal { + toolUseId: string; + content: string; + isError: boolean; +} + +export interface ExecutionSignals { + totalMessages: number; + totalCharacters: number; + toolCalls: ToolCallSignal[]; + toolResults: ToolResultSignal[]; + + // Heuristics + filesTouched: string[]; + repeatedEdits: boolean; + hasTestFailures: boolean; + consecutiveFailures: number; +} + +/** Case-insensitive keys to search for file paths in tool arguments */ +const FILE_PATH_KEYS = ["path", "filepath", "file", "filename", "targetfile", "absolutepath"]; + +function extractFilePath(args: unknown): string | null { + if (!args) return null; + let obj: Record = {}; + if (typeof args === "string") { + try { + obj = JSON.parse(args); + } catch { + return null; + } + } else if (typeof args === "object" && args !== null) { + obj = args as Record; + } + + for (const [k, val] of Object.entries(obj)) { + if (FILE_PATH_KEYS.includes(k.toLowerCase()) && typeof val === "string" && val.length > 0) { + return val; + } + } + return null; +} + +/** + * Word-boundary matching avoids substring false positives ("failover", "failsafe" + * are not failures). `fail` variants, `error:`, assertion/exit/command markers count. + */ +const FAILURE_RE = /\bfail(ed|ing|ure|s)?\b|error:|assertion\s?error|exit status|command failed/i; + +function isFailureContent(content: string): boolean { + return FAILURE_RE.test(content); +} + +/** A failure result that specifically points at a test run. */ +const TEST_FAILURE_RE = /\btests?\b|\bfail(ed|ing|ure|s)?\b/i; + +export function extractSignals(req: ChatRequest): ExecutionSignals { + const signals: ExecutionSignals = { + totalMessages: req.messages.length, + totalCharacters: 0, + toolCalls: [], + toolResults: [], + filesTouched: [], + repeatedEdits: false, + hasTestFailures: false, + consecutiveFailures: 0, + }; + + // 1. Core Metrics & Base Text Extraction + for (const msg of req.messages) { + signals.totalCharacters += msg.content.length; + } + + const raw = req.rawBody as any; + if (!raw) return signals; + + // 2. Parse Tool Calls & Results per Dialect + if (req.dialect === "openai") { + const messages = raw.messages ?? []; + for (const m of messages) { + // Extract Tool Calls + if (m.role === "assistant" && Array.isArray(m.tool_calls)) { + for (const tc of m.tool_calls) { + if (tc.type === "function" && tc.function) { + signals.toolCalls.push({ + name: tc.function.name, + arguments: tc.function.arguments, + }); + } + } + } + // Extract Tool Results + if (m.role === "tool") { + const content = typeof m.content === "string" ? m.content : JSON.stringify(m.content ?? ""); + signals.toolResults.push({ + toolUseId: m.tool_call_id ?? "", + content, + isError: isFailureContent(content), + }); + } + } + } else if (req.dialect === "anthropic") { + const messages = raw.messages ?? []; + for (const m of messages) { + if (Array.isArray(m.content)) { + for (const part of m.content) { + // Extract Tool Calls + if (m.role === "assistant" && part.type === "tool_use") { + signals.toolCalls.push({ + name: part.name, + arguments: part.input, + }); + } + // Extract Tool Results + if (m.role === "user" && part.type === "tool_result") { + let contentStr = ""; + if (typeof part.content === "string") { + contentStr = part.content; + } else if (Array.isArray(part.content)) { + contentStr = part.content + .map((c: any) => (c && typeof c === "object" && c.type === "text" ? String(c.text ?? "") : "")) + .join(""); + } + signals.toolResults.push({ + toolUseId: part.tool_use_id ?? "", + content: contentStr, + isError: part.is_error === true || isFailureContent(contentStr), + }); + } + } + } + } + } + + // 3. Compute Heuristics (Files Touched & Repeated Edits) + const editTools = ["write", "edit", "replace", "save", "patch", "modify", "str_replace_editor"]; + const fileEdits: string[] = []; + + for (const tc of signals.toolCalls) { + const path = extractFilePath(tc.arguments); + if (path) { + if (!signals.filesTouched.includes(path)) { + signals.filesTouched.push(path); + } + const lowerName = tc.name.toLowerCase(); + if (editTools.some((t) => lowerName.includes(t))) { + fileEdits.push(path); + } + } + } + + // Detect repeated edits: did we edit the same file multiple times? + if (fileEdits.length >= 2) { + for (let i = 0; i < fileEdits.length - 1; i++) { + if (fileEdits[i] === fileEdits[i + 1]) { + signals.repeatedEdits = true; + break; + } + } + } + + // 4. Compute Failures (Consecutive failures & hasTestFailures) + let consecutive = 0; + for (const res of signals.toolResults) { + if (res.isError) { + consecutive++; + if (TEST_FAILURE_RE.test(res.content)) { + signals.hasTestFailures = true; + } + } else { + consecutive = 0; + } + } + signals.consecutiveFailures = consecutive; + + return signals; +}