From 1a67bd4611313d99fd580cfec7d55fb316d55d6f Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Thu, 13 Aug 2026 22:37:05 +0900 Subject: [PATCH 1/4] test(responses): pin tool round-trip conformance across both transports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opens the PR-B conformance layer (devlog 030-034). The plan's premise was that a translator reading only top-level `tools` silently erases terminal, custom and namespace tools, after which the model emits ordinary text and completes normally — a protocol loss that looks like model behavior. A read-only inventory of this checkout found that premise is partly obsolete here: `additional_tools` IS parsed and merged. So this suite pins where the risk actually lives. New shared harness, tests/helpers/responses-conformance.ts. Every existing tool test re-implements `replay`/`collectSse` locally, which is exactly why the streaming and non-streaming paths had never been compared: each test only ever looked at one of them. The harness pushes one fixture through BOTH bridges and normalizes `arguments`/`input` into a single comparable shape, reading the streamed side from `response.completed` because that is what a client which reconnects or ignores deltas actually sees. Coverage, 19 cases: - additional_tools merge: top+nested, nested-only (the Codex Desktop responses_lite shape whose tool surface rides inside `input`), multiple groups in wire order, and top-level winning a qualified-name collision; - kind discrimination: function/custom/tool_search/namespace markers; - tool_search history: a discovered tool becomes callable on the next turn, including a namespaced one, or deferred discovery is a one-way trip; - transport parity across function, custom with split escapes and non-ASCII, namespaced, tool_search, and text-before-call. Four cases deliberately pin CURRENT degradation rather than desired behavior, labeled as such so changing them is a visible decision rather than a surprise: a malformed `additional_tools` item is ignored instead of rejected (031 asks for explicit failure); an unknown NAMED kind survives only as a callable function with its kind unrecoverable; an unknown UNNAMED kind disappears; and a non-function child inside a namespace disappears. The parallel-call test pins a contract limitation, not a bug: `tool_call_start` carries an id but `tool_call_delta`/`tool_call_end` do not (src/types.ts:323), so the bridge tracks one call at a time and interleaved fragments are split by arrival order rather than by call id. Fixing the event contract should make that assertion fail. Ablations proving the suite is not vacuous: - disabling the `additional_tools` branch in the parser fails 3 cases — the exact silent-loss class the plan feared; - collapsing the non-streaming custom restoration to `function_call` fails the parity case, a divergence no per-path test could see. Verification: bun x tsc --noEmit clean; 155 pass / 0 fail across the 6 tool suites; privacy:scan passed. --- tests/helpers/responses-conformance.ts | 110 ++++++++ tests/responses-tool-conformance.test.ts | 305 +++++++++++++++++++++++ 2 files changed, 415 insertions(+) create mode 100644 tests/helpers/responses-conformance.ts create mode 100644 tests/responses-tool-conformance.test.ts diff --git a/tests/helpers/responses-conformance.ts b/tests/helpers/responses-conformance.ts new file mode 100644 index 000000000..4e95d48c2 --- /dev/null +++ b/tests/helpers/responses-conformance.ts @@ -0,0 +1,110 @@ +import { bridgeToResponsesSSE, buildResponseJSON } from "../../src/bridge"; +import type { AdapterEvent } from "../../src/types"; + +/** + * Shared harness for Responses tool round-trip conformance + * (devlog/_plan/260813_routed_tool_discovery_profiles/030-034). + * + * Every existing tool test re-implements `replay` and `collectSse` locally, which is why the + * streaming and non-streaming paths have never been compared against each other on a common + * shape: each test only ever looked at one of them. This module exists so one fixture can be + * pushed through BOTH bridges and normalized into a single comparable structure. + */ + +export async function* replay(events: readonly AdapterEvent[]): AsyncGenerator { + for (const event of events) yield event; +} + +export interface SseFrame { + event?: string; + data: Record; +} + +export async function collectSse(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + return text.split("\n\n") + .map(frame => frame.trim()) + .filter(frame => frame.length > 0 && frame !== "data: [DONE]") + .map(frame => { + const lines = frame.split("\n"); + const event = lines.find(line => line.startsWith("event: "))?.slice(7); + const dataLine = lines.find(line => line.startsWith("data: ")); + return { event, data: JSON.parse(dataLine?.slice(6) ?? "{}") as Record }; + }); +} + +/** The tool-bearing fields both transports must agree on, in output order. */ +export interface NormalizedToolItem { + type: string; + name?: string; + call_id?: string; + /** `arguments` for function/tool_search, `input` for custom — normalized to one field. */ + payload?: string; + status?: string; +} + +function normalizeItem(item: Record): NormalizedToolItem { + const payload = typeof item.arguments === "string" + ? item.arguments + : typeof item.input === "string" ? item.input : undefined; + return { + type: String(item.type ?? ""), + ...(typeof item.name === "string" ? { name: item.name } : {}), + ...(typeof item.call_id === "string" ? { call_id: item.call_id } : {}), + ...(payload !== undefined ? { payload } : {}), + ...(typeof item.status === "string" ? { status: item.status } : {}), + }; +} + +type BridgeMaps = [ + toolNsMap?: Map, + freeformToolNames?: Set, + toolSearchToolNames?: Set, +]; + +/** + * Tool items from the streamed transport, read from `response.completed`. + * + * Deliberately NOT read from `output_item.done` frames: the completed snapshot is what a + * client that reconnects or ignores deltas actually sees, so it is the honest counterpart to + * the non-streaming body. + */ +export async function streamedToolItems( + events: readonly AdapterEvent[], + modelId: string, + ...maps: BridgeMaps +): Promise { + const frames = await collectSse(bridgeToResponsesSSE(replay(events), modelId, ...maps)); + const completed = frames.find(frame => frame.event === "response.completed"); + const response = completed?.data.response as Record | undefined; + const output = Array.isArray(response?.output) ? response.output as Record[] : []; + return output.filter(item => String(item.type ?? "").includes("call")).map(normalizeItem); +} + +/** Tool items from the non-streaming transport. */ +export function jsonToolItems( + events: readonly AdapterEvent[], + modelId: string, + options?: Parameters[2], +): NormalizedToolItem[] { + const body = buildResponseJSON([...events], modelId, options); + const output = Array.isArray(body.output) ? body.output as Record[] : []; + return output.filter(item => String(item.type ?? "").includes("call")).map(normalizeItem); +} + +/** All frame event names in order — for delta-level behavior a snapshot cannot show. */ +export async function streamedEventNames( + events: readonly AdapterEvent[], + modelId: string, + ...maps: BridgeMaps +): Promise { + const frames = await collectSse(bridgeToResponsesSSE(replay(events), modelId, ...maps)); + return frames.map(frame => frame.event ?? ""); +} diff --git a/tests/responses-tool-conformance.test.ts b/tests/responses-tool-conformance.test.ts new file mode 100644 index 000000000..a046ffd45 --- /dev/null +++ b/tests/responses-tool-conformance.test.ts @@ -0,0 +1,305 @@ +import { describe, expect, it } from "bun:test"; +import { parseRequest } from "../src/responses/parser"; +import type { AdapterEvent } from "../src/types"; +import { jsonToolItems, streamedEventNames, streamedToolItems } from "./helpers/responses-conformance"; + +/** + * Responses tool round-trip conformance + * (devlog/_plan/260813_routed_tool_discovery_profiles/030-034). + * + * The plan's premise was that a translator reading only top-level `tools` can silently erase + * terminal, custom and namespace tools, and that the model then emits ordinary text and + * completes normally — a failure that looks like model behavior rather than protocol loss. + * This suite pins where that risk actually lives in THIS codebase, which is not where the + * plan assumed: `additional_tools` is handled, so the residual exposure is malformed input, + * unknown tool kinds, and stream/non-stream divergence. + * + * Several cases below deliberately pin CURRENT degradation rather than desired behavior. + * They are labeled as such, so a future change to that behavior fails here loudly instead of + * being discovered by a user whose tool vanished. + */ + +const MODEL = "deepseek/glm-5.2"; + +function request(input: unknown[], tools?: unknown[]): Record { + return { + model: MODEL, + input, + ...(tools ? { tools } : {}), + }; +} + +function toolNames(parsed: ReturnType): string[] { + return (parsed.context.tools ?? []).map(tool => tool.name); +} + +describe("Responses Lite additional_tools declaration merge", () => { + const fnTool = { type: "function", name: "read_file", parameters: { type: "object", properties: {} } }; + const nsTool = { + type: "namespace", + name: "github", + tools: [{ type: "function", name: "search", parameters: { type: "object", properties: {} } }], + }; + const customTool = { type: "custom", name: "apply_patch" }; + + it("merges top-level tools and additional_tools input items", () => { + const parsed = parseRequest(request( + [{ type: "additional_tools", role: "developer", tools: [customTool] }], + [fnTool], + )); + expect(toolNames(parsed)).toEqual(["read_file", "apply_patch"]); + }); + + it("accepts an additional_tools-only request, the Codex Desktop responses_lite shape", () => { + // The tool surface rides INSIDE input rather than body.tools. A translator that reads + // only body.tools sees an empty catalog and the turn silently loses every tool. + const parsed = parseRequest(request( + [{ type: "additional_tools", role: "developer", tools: [fnTool, nsTool, customTool] }], + )); + // Namespaced children keep their BARE name; the namespace rides alongside on the tool. + expect(toolNames(parsed)).toEqual(["read_file", "search", "apply_patch"]); + expect(parsed.context.tools?.find(tool => tool.name === "search")?.namespace).toBe("github"); + }); + + it("preserves wire order across multiple additional_tools groups", () => { + const parsed = parseRequest(request([ + { type: "additional_tools", role: "developer", tools: [fnTool] }, + { type: "additional_tools", role: "developer", tools: [customTool] }, + ])); + expect(toolNames(parsed)).toEqual(["read_file", "apply_patch"]); + }); + + it("lets the top-level declaration win a qualified-name collision", () => { + const shadowed = { type: "function", name: "read_file", parameters: { type: "object", properties: { shadow: { type: "string" } } } }; + const parsed = parseRequest(request( + [{ type: "additional_tools", role: "developer", tools: [shadowed] }], + [fnTool], + )); + expect(toolNames(parsed)).toEqual(["read_file"]); + // The surviving entry is the TOP-LEVEL one, not the nested shadow. + expect(parsed.context.tools?.[0]?.parameters).toEqual(fnTool.parameters as never); + }); + + it("CURRENT BEHAVIOR: a malformed additional_tools item is ignored, not rejected", () => { + // devlog 031 asks for explicit failure here. Today the item is skipped silently, so a + // typo in the tool surface degrades to "model has no tools" with no diagnostic. Pinned + // so that changing it to a hard error is a visible, deliberate decision. + const parsed = parseRequest(request([ + { type: "additional_tools", role: "developer", tools: "not-an-array" }, + ])); + expect(parsed.context.tools ?? []).toEqual([]); + }); +}); + +describe("Responses tool-kind discrimination", () => { + it("maps each known kind onto its internal marker", () => { + const parsed = parseRequest(request([], [ + { type: "function", name: "fn", parameters: { type: "object", properties: {} } }, + { type: "custom", name: "freeform" }, + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + { type: "namespace", name: "ns", tools: [{ type: "function", name: "child", parameters: { type: "object", properties: {} } }] }, + ])); + const byName = new Map((parsed.context.tools ?? []).map(tool => [tool.name, tool])); + expect(byName.get("fn")?.freeform).toBeUndefined(); + expect(byName.get("freeform")?.freeform).toBe(true); + expect(byName.get("tool_search")?.toolSearch).toBe(true); + expect(byName.get("child")?.namespace).toBe("ns"); + }); + + it("CURRENT BEHAVIOR: an unknown NAMED kind survives as a callable function", () => { + // Better than the historical silent drop, but the original kind is not recoverable, so + // the response cannot be restored as that kind either. + const parsed = parseRequest(request([], [ + { type: "computer_use_preview", name: "computer", parameters: { type: "object", properties: {} } }, + ])); + expect(toolNames(parsed)).toEqual(["computer"]); + expect(parsed.context.tools?.[0]?.freeform).toBeUndefined(); + }); + + it("CURRENT BEHAVIOR: an unknown UNNAMED kind disappears entirely", () => { + // This is the plan's silent-loss class, still live. There is no name to pass through, so + // the declaration is dropped with no diagnostic. + const parsed = parseRequest(request([], [ + { type: "some_future_hosted_tool", config: { enabled: true } }, + ])); + expect(parsed.context.tools ?? []).toEqual([]); + }); + + it("CURRENT BEHAVIOR: a non-function child inside a namespace disappears", () => { + const parsed = parseRequest(request([], [ + { + type: "namespace", + name: "ns", + tools: [ + { type: "function", name: "kept", parameters: { type: "object", properties: {} } }, + { type: "custom", name: "dropped" }, + ], + }, + ])); + expect(toolNames(parsed)).toEqual(["kept"]); + }); +}); + +describe("tool_search call and output history", () => { + it("loads definitions returned by a tool_search_output into the active catalog", () => { + const parsed = parseRequest(request([ + { type: "message", role: "user", content: [{ type: "input_text", text: "find it" }] }, + { type: "tool_search_call", id: "ts_1", call_id: "ts_1", execution: "client", arguments: "{\"query\":\"repl\"}", status: "completed" }, + { + type: "tool_search_output", + call_id: "ts_1", + status: "completed", + execution: "client", + tools: [{ type: "function", name: "node_repl", parameters: { type: "object", properties: {} } }], + }, + ], [ + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + ])); + + // The discovered tool must be callable on the NEXT turn, or deferred discovery is a + // one-way trip and the model can never invoke what it just found. + expect(toolNames(parsed)).toContain("node_repl"); + const loaded = parsed.context.tools?.find(tool => tool.name === "node_repl"); + expect(loaded?.loadedFromToolSearch).toBe(true); + }); + + it("flattens a namespaced tool discovered through tool_search", () => { + const parsed = parseRequest(request([ + { + type: "tool_search_output", + call_id: "ts_2", + status: "completed", + execution: "client", + tools: [{ + type: "namespace", + name: "browser", + tools: [{ type: "function", name: "open", parameters: { type: "object", properties: {} } }], + }], + }, + ], [ + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + ])); + expect(toolNames(parsed)).toContain("open"); + expect(parsed.context.tools?.find(tool => tool.name === "open")?.namespace).toBe("browser"); + }); +}); + +describe("streaming and non-streaming tool parity", () => { + const nsMap = new Map([["ns__child", { namespace: "ns", name: "child" }]]); + const freeform = new Set(["apply_patch"]); + const toolSearch = new Set(["tool_search"]); + + const cases: Array<{ label: string; events: AdapterEvent[] }> = [ + { + label: "function call", + events: [ + { type: "tool_call_start", id: "call_fn", name: "read_file" }, + { type: "tool_call_delta", arguments: "{\"path\"" }, + { type: "tool_call_delta", arguments: ":\"a.txt\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "custom/freeform call with split escapes and non-ASCII", + events: [ + { type: "tool_call_start", id: "call_custom", name: "apply_patch" }, + { type: "tool_call_delta", arguments: "{\"input\":\"안녕 \\" }, + { type: "tool_call_delta", arguments: "\"quoted\\\" 世界\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "namespaced call", + events: [ + { type: "tool_call_start", id: "call_ns", name: "ns__child" }, + { type: "tool_call_delta", arguments: "{}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "tool_search call", + events: [ + { type: "tool_call_start", id: "call_ts", name: "tool_search" }, + { type: "tool_call_delta", arguments: "{\"query\":\"repl\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "text before a call", + events: [ + { type: "text_delta", text: "working" }, + { type: "tool_call_start", id: "call_after_text", name: "read_file" }, + { type: "tool_call_delta", arguments: "{}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + ]; + + for (const { label, events } of cases) { + it(`agrees between transports for a ${label}`, async () => { + const streamed = await streamedToolItems(events, MODEL, nsMap, freeform, toolSearch); + const json = jsonToolItems(events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + // Same restored item TYPE, name, call id and payload on both transports. A divergence + // here is the "restored as the wrong event" failure class from devlog 034. + expect(streamed).toEqual(json); + }); + } + + it("restores each kind as its own item type rather than collapsing to function_call", async () => { + const kinds = await Promise.all(cases.map(async ({ events }) => { + const [item] = await streamedToolItems(events, MODEL, nsMap, freeform, toolSearch); + return item?.type; + })); + expect(kinds).toEqual([ + "function_call", + "custom_tool_call", + "function_call", + "tool_search_call", + "function_call", + ]); + }); + + it("emits custom input deltas on the streamed path only", async () => { + const custom = cases.find(entry => entry.label.startsWith("custom"))!; + const names = await streamedEventNames(custom.events, MODEL, nsMap, freeform, toolSearch); + expect(names).toContain("response.custom_tool_call_input.delta"); + expect(names).not.toContain("response.function_call_arguments.delta"); + }); +}); + +describe("parallel tool-call capability", () => { + it("CURRENT LIMITATION: interleaved calls cannot be represented by AdapterEvent", async () => { + // `tool_call_start` carries an id, but `tool_call_delta` and `tool_call_end` do not + // (src/types.ts). The bridge therefore tracks ONE call at a time, so a provider that + // interleaves two calls has no way to say which fragment belongs to which. + // + // This test pins the consequence rather than pretending it works: the second start + // closes the first call, and the interleaved fragments are NOT reunited with their + // owners. Fixing the event contract should make this assertion fail. + const interleaved: AdapterEvent[] = [ + { type: "tool_call_start", id: "call_a", name: "read_file" }, + { type: "tool_call_delta", arguments: "{\"path\":\"a" }, + { type: "tool_call_start", id: "call_b", name: "read_file" }, + { type: "tool_call_delta", arguments: "{\"path\":\"b.txt\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ]; + + const streamed = await streamedToolItems(interleaved, MODEL); + const json = jsonToolItems(interleaved, MODEL); + // Both transports agree with each other, which is what makes this a contract limitation + // rather than a transport bug. + expect(streamed).toEqual(json); + + const payloads = streamed.map(item => item.payload); + // call_a's fragment did not receive call_b's continuation: the arguments are split by + // ARRIVAL ORDER, not by call id. + expect(payloads[0]).toBe("{\"path\":\"a"); + expect(streamed.map(item => item.call_id)).toEqual(["call_a", "call_b"]); + }); +}); From eba7974ef1cc1ec0284f497d59117d5440399e44 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Thu, 13 Aug 2026 22:48:49 +0900 Subject: [PATCH 2/4] test(responses): compare incremental frames, not just the final snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An independent audit found four of my conformance tests could not fail, and the parity harness hid a whole divergence class. Every finding reproduced. The harness read only `response.completed`. The bridge builds `response.output_item.done` separately, so corrupting ONLY the incremental item type left all 19 tests green: a client consuming normal frames would see the wrong type while a reconnecting client saw the right one. `streamedView()` now returns snapshot, incremental, event names, ordered deltas and non-call item types, and every parity case asserts all three surfaces agree. `NormalizedToolItem` gained `namespace` and now keeps object payloads verbatim. Both omissions were silent holes: deleting namespace restoration from both bridges passed, and so did replacing tool_search arguments with `{}` — the normalizer only accepted string payloads, so an object payload was simply dropped from the comparison. Fixed vacuous cases: - malformed `additional_tools` now carries a well-formed sibling group, so it distinguishes "one bad item ignored" from "feature deleted entirely"; - the kind-marker case asserts the key SET first, since `byName.get("fn")?.freeform` is also undefined when `fn` is missing; - text-before-call asserts the message item survives on both transports, since the tool filter hid it and made the case a duplicate function-call test; - the custom-delta case asserts exact ordered fragments reassemble into the final payload and that exactly one terminal item event is emitted, rather than only checking an event name. The parallel case is now a genuine A/B/A/B interleaving: A is fragmented, B starts mid-flight, then A's continuation arrives and is misattributed to B, because fragments route by arrival order rather than call id. Ablations, each previously green and now red: incremental item type (6 fail), namespace restoration (2), tool_search arguments (1), dropped assistant text (1), plus the two from the prior commit. SCOPE, stated plainly: this starts the PR-B layer, it does not complete it. Still absent from devlog 030-034 — the end-to-end declaration/call/result/second -turn execution loop, compaction and resume with a discovered tool, the full collision matrix, per-adapter declaration comparison, transport-error parity, and the machine-readable conformance artifacts. Verification: bun x tsc --noEmit clean; 158 pass / 0 fail across the 6 tool suites; privacy:scan passed. --- tests/helpers/responses-conformance.ts | 83 +++++++++++------ tests/responses-tool-conformance.test.ts | 109 +++++++++++++++++------ 2 files changed, 137 insertions(+), 55 deletions(-) diff --git a/tests/helpers/responses-conformance.ts b/tests/helpers/responses-conformance.ts index 4e95d48c2..0991a7afd 100644 --- a/tests/helpers/responses-conformance.ts +++ b/tests/helpers/responses-conformance.ts @@ -5,10 +5,14 @@ import type { AdapterEvent } from "../../src/types"; * Shared harness for Responses tool round-trip conformance * (devlog/_plan/260813_routed_tool_discovery_profiles/030-034). * - * Every existing tool test re-implements `replay` and `collectSse` locally, which is why the - * streaming and non-streaming paths have never been compared against each other on a common - * shape: each test only ever looked at one of them. This module exists so one fixture can be - * pushed through BOTH bridges and normalized into a single comparable structure. + * Every existing tool test re-implements `replay`/`collectSse` locally, which is why the + * streaming and non-streaming paths had never been compared: each test only looked at one. + * + * The streamed side is read from BOTH surfaces on purpose. `response.completed` is what a + * client that reconnects or ignores deltas sees; `response.output_item.done` is what a client + * consuming normal incremental frames sees. The bridge builds them separately, so reading only + * the snapshot hides a whole divergence class — an item can be correct in the final snapshot + * and wrong in the incremental frame. devlog 034 requires the incremental assertions. */ export async function* replay(events: readonly AdapterEvent[]): AsyncGenerator { @@ -40,52 +44,76 @@ export async function collectSse(stream: ReadableStream): Promise): NormalizedToolItem { - const payload = typeof item.arguments === "string" - ? item.arguments - : typeof item.input === "string" ? item.input : undefined; + const payload = item.arguments !== undefined ? item.arguments : item.input; return { type: String(item.type ?? ""), ...(typeof item.name === "string" ? { name: item.name } : {}), ...(typeof item.call_id === "string" ? { call_id: item.call_id } : {}), ...(payload !== undefined ? { payload } : {}), ...(typeof item.status === "string" ? { status: item.status } : {}), + ...(typeof item.namespace === "string" ? { namespace: item.namespace } : {}), }; } +const isToolItem = (item: Record): boolean => + String(item.type ?? "").includes("call"); + type BridgeMaps = [ toolNsMap?: Map, freeformToolNames?: Set, toolSearchToolNames?: Set, ]; -/** - * Tool items from the streamed transport, read from `response.completed`. - * - * Deliberately NOT read from `output_item.done` frames: the completed snapshot is what a - * client that reconnects or ignores deltas actually sees, so it is the honest counterpart to - * the non-streaming body. - */ -export async function streamedToolItems( +export interface StreamedView { + /** Tool items from the terminal `response.completed` snapshot. */ + snapshot: NormalizedToolItem[]; + /** Tool items from the incremental `response.output_item.done` frames. */ + incremental: NormalizedToolItem[]; + /** Every frame's event name, in order. */ + eventNames: string[]; + /** Ordered payloads of every argument/input delta frame. */ + deltas: string[]; + /** Every non-call output item type from the snapshot, e.g. "message". */ + snapshotItemTypes: string[]; +} + +export async function streamedView( events: readonly AdapterEvent[], modelId: string, ...maps: BridgeMaps -): Promise { +): Promise { const frames = await collectSse(bridgeToResponsesSSE(replay(events), modelId, ...maps)); const completed = frames.find(frame => frame.event === "response.completed"); const response = completed?.data.response as Record | undefined; const output = Array.isArray(response?.output) ? response.output as Record[] : []; - return output.filter(item => String(item.type ?? "").includes("call")).map(normalizeItem); + + const doneItems = frames + .filter(frame => frame.event === "response.output_item.done") + .map(frame => frame.data.item) + .filter((item): item is Record => !!item && typeof item === "object"); + + return { + snapshot: output.filter(isToolItem).map(normalizeItem), + incremental: doneItems.filter(isToolItem).map(normalizeItem), + eventNames: frames.map(frame => frame.event ?? ""), + deltas: frames + .filter(frame => frame.event?.endsWith(".delta") && typeof frame.data.delta === "string") + .map(frame => String(frame.data.delta)), + snapshotItemTypes: output.map(item => String(item.type ?? "")), + }; } /** Tool items from the non-streaming transport. */ @@ -96,15 +124,16 @@ export function jsonToolItems( ): NormalizedToolItem[] { const body = buildResponseJSON([...events], modelId, options); const output = Array.isArray(body.output) ? body.output as Record[] : []; - return output.filter(item => String(item.type ?? "").includes("call")).map(normalizeItem); + return output.filter(isToolItem).map(normalizeItem); } -/** All frame event names in order — for delta-level behavior a snapshot cannot show. */ -export async function streamedEventNames( +/** Every output item type from the non-streaming transport, including non-call items. */ +export function jsonItemTypes( events: readonly AdapterEvent[], modelId: string, - ...maps: BridgeMaps -): Promise { - const frames = await collectSse(bridgeToResponsesSSE(replay(events), modelId, ...maps)); - return frames.map(frame => frame.event ?? ""); + options?: Parameters[2], +): string[] { + const body = buildResponseJSON([...events], modelId, options); + const output = Array.isArray(body.output) ? body.output as Record[] : []; + return output.map(item => String(item.type ?? "")); } diff --git a/tests/responses-tool-conformance.test.ts b/tests/responses-tool-conformance.test.ts index a046ffd45..7d68086a1 100644 --- a/tests/responses-tool-conformance.test.ts +++ b/tests/responses-tool-conformance.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "bun:test"; import { parseRequest } from "../src/responses/parser"; import type { AdapterEvent } from "../src/types"; -import { jsonToolItems, streamedEventNames, streamedToolItems } from "./helpers/responses-conformance"; +import { jsonItemTypes, jsonToolItems, streamedView } from "./helpers/responses-conformance"; /** * Responses tool round-trip conformance @@ -84,10 +84,15 @@ describe("Responses Lite additional_tools declaration merge", () => { // devlog 031 asks for explicit failure here. Today the item is skipped silently, so a // typo in the tool surface degrades to "model has no tools" with no diagnostic. Pinned // so that changing it to a hard error is a visible, deliberate decision. + // + // A WELL-FORMED sibling group rides along deliberately: without it this case would also + // pass if additional_tools support were deleted outright, and "ignored one bad item" + // would be indistinguishable from "lost the whole feature". const parsed = parseRequest(request([ { type: "additional_tools", role: "developer", tools: "not-an-array" }, + { type: "additional_tools", role: "developer", tools: [fnTool] }, ])); - expect(parsed.context.tools ?? []).toEqual([]); + expect(toolNames(parsed)).toEqual(["read_file"]); }); }); @@ -100,6 +105,9 @@ describe("Responses tool-kind discrimination", () => { { type: "namespace", name: "ns", tools: [{ type: "function", name: "child", parameters: { type: "object", properties: {} } }] }, ])); const byName = new Map((parsed.context.tools ?? []).map(tool => [tool.name, tool])); + // Presence first: `byName.get("fn")?.freeform` is also undefined when "fn" is MISSING, + // so the marker assertion alone would survive deleting the function branch. + expect([...byName.keys()].sort()).toEqual(["child", "fn", "freeform", "tool_search"]); expect(byName.get("fn")?.freeform).toBeUndefined(); expect(byName.get("freeform")?.freeform).toBe(true); expect(byName.get("tool_search")?.toolSearch).toBe(true); @@ -241,19 +249,55 @@ describe("streaming and non-streaming tool parity", () => { ]; for (const { label, events } of cases) { - it(`agrees between transports for a ${label}`, async () => { - const streamed = await streamedToolItems(events, MODEL, nsMap, freeform, toolSearch); + it(`agrees across snapshot, incremental frames and JSON for a ${label}`, async () => { + const view = await streamedView(events, MODEL, nsMap, freeform, toolSearch); const json = jsonToolItems(events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); - // Same restored item TYPE, name, call id and payload on both transports. A divergence - // here is the "restored as the wrong event" failure class from devlog 034. - expect(streamed).toEqual(json); + + // Three surfaces, not two. `response.completed` is what a reconnecting client sees; + // `output_item.done` is what a client consuming normal incremental frames sees. The + // bridge builds them separately, so comparing only the snapshot hides an item that is + // correct at the end and wrong on the wire (devlog 034). + expect(view.incremental).toEqual(view.snapshot); + expect(view.snapshot).toEqual(json); }); } + it("preserves assistant text alongside the call on both transports", async () => { + // The tool-item filter hides non-call output, so without this the "text before a call" + // fixture would be just another function-call parity case. + const textCase = cases.find(entry => entry.label.startsWith("text"))!; + const view = await streamedView(textCase.events, MODEL, nsMap, freeform, toolSearch); + const jsonTypes = jsonItemTypes(textCase.events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + expect(view.snapshotItemTypes).toContain("message"); + expect(jsonTypes).toContain("message"); + expect(view.snapshotItemTypes).toEqual(jsonTypes); + }); + + it("restores namespace identity identically on both transports", async () => { + // NormalizedToolItem carries `namespace`, so dropping namespace restoration from either + // bridge fails here. An earlier revision omitted the field and could not see it at all. + const nsCase = cases.find(entry => entry.label.startsWith("namespaced"))!; + const view = await streamedView(nsCase.events, MODEL, nsMap, freeform, toolSearch); + const json = jsonToolItems(nsCase.events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + expect(view.snapshot[0]?.name).toBe("child"); + expect(json[0]?.name).toBe("child"); + expect(view.snapshot).toEqual(json); + }); + + it("carries the tool_search payload rather than an empty object", async () => { + // tool_search arguments may be an object rather than a string; the normalizer keeps the + // value verbatim so replacing it with {} on both paths cannot pass silently. + const tsCase = cases.find(entry => entry.label.startsWith("tool_search"))!; + const view = await streamedView(tsCase.events, MODEL, nsMap, freeform, toolSearch); + const json = jsonToolItems(tsCase.events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + expect(JSON.stringify(view.snapshot[0]?.payload)).toContain("repl"); + expect(view.snapshot).toEqual(json); + }); + it("restores each kind as its own item type rather than collapsing to function_call", async () => { const kinds = await Promise.all(cases.map(async ({ events }) => { - const [item] = await streamedToolItems(events, MODEL, nsMap, freeform, toolSearch); - return item?.type; + const view = await streamedView(events, MODEL, nsMap, freeform, toolSearch); + return view.snapshot[0]?.type; })); expect(kinds).toEqual([ "function_call", @@ -264,42 +308,51 @@ describe("streaming and non-streaming tool parity", () => { ]); }); - it("emits custom input deltas on the streamed path only", async () => { + it("emits the exact custom input fragments on the streamed path only", async () => { const custom = cases.find(entry => entry.label.startsWith("custom"))!; - const names = await streamedEventNames(custom.events, MODEL, nsMap, freeform, toolSearch); - expect(names).toContain("response.custom_tool_call_input.delta"); - expect(names).not.toContain("response.function_call_arguments.delta"); + const view = await streamedView(custom.events, MODEL, nsMap, freeform, toolSearch); + expect(view.eventNames).toContain("response.custom_tool_call_input.delta"); + expect(view.eventNames).not.toContain("response.function_call_arguments.delta"); + // Exact ordered fragments, not just the event name: a corrupted, duplicated or reordered + // delta stream would otherwise pass. + expect(view.deltas.join("")).toBe(String(view.snapshot[0]?.payload ?? "")); + // Exactly one terminal item event for the one call. + expect(view.eventNames.filter(name => name === "response.output_item.done")).toHaveLength(1); }); }); describe("parallel tool-call capability", () => { it("CURRENT LIMITATION: interleaved calls cannot be represented by AdapterEvent", async () => { // `tool_call_start` carries an id, but `tool_call_delta` and `tool_call_end` do not - // (src/types.ts). The bridge therefore tracks ONE call at a time, so a provider that + // (src/types.ts:323). Both bridges therefore track ONE current call, so a provider that // interleaves two calls has no way to say which fragment belongs to which. // - // This test pins the consequence rather than pretending it works: the second start - // closes the first call, and the interleaved fragments are NOT reunited with their - // owners. Fixing the event contract should make this assertion fail. + // A genuine A/B/A/B interleaving: call A is fragmented, B starts mid-flight, then A's + // continuation arrives. This pins the consequence rather than pretending it works — + // A's later fragment is misattributed to B, because fragments are routed by ARRIVAL + // ORDER, not by call id. Giving delta/end a call id should make this fail. const interleaved: AdapterEvent[] = [ { type: "tool_call_start", id: "call_a", name: "read_file" }, { type: "tool_call_delta", arguments: "{\"path\":\"a" }, - { type: "tool_call_start", id: "call_b", name: "read_file" }, - { type: "tool_call_delta", arguments: "{\"path\":\"b.txt\"}" }, + { type: "tool_call_start", id: "call_b", name: "write_file" }, + { type: "tool_call_delta", arguments: "{\"path\":\"b" }, + { type: "tool_call_delta", arguments: ".txt\"}" }, { type: "tool_call_end" }, { type: "done" }, ]; - const streamed = await streamedToolItems(interleaved, MODEL); + const view = await streamedView(interleaved, MODEL); const json = jsonToolItems(interleaved, MODEL); - // Both transports agree with each other, which is what makes this a contract limitation - // rather than a transport bug. - expect(streamed).toEqual(json); + // Both transports agree with each other AND with the incremental frames, which is what + // makes this a contract limitation rather than a transport bug. + expect(view.incremental).toEqual(view.snapshot); + expect(view.snapshot).toEqual(json); - const payloads = streamed.map(item => item.payload); - // call_a's fragment did not receive call_b's continuation: the arguments are split by - // ARRIVAL ORDER, not by call id. - expect(payloads[0]).toBe("{\"path\":\"a"); - expect(streamed.map(item => item.call_id)).toEqual(["call_a", "call_b"]); + expect(view.snapshot.map(item => item.call_id)).toEqual(["call_a", "call_b"]); + expect(view.snapshot.map(item => item.name)).toEqual(["read_file", "write_file"]); + // A keeps only what arrived before B started; B absorbs A's continuation. Neither + // payload is the JSON its provider actually sent. + expect(view.snapshot[0]?.payload).toBe("{\"path\":\"a"); + expect(view.snapshot[1]?.payload).toBe("{\"path\":\"b.txt\"}"); }); }); From a6eb6e50c9b3116ec87ee24121a3c16a1d2e5eb8 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Thu, 13 Aug 2026 22:56:14 +0900 Subject: [PATCH 3/4] test(responses): close three more vacuous conformance assertions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-audit found my own ablation report contained a false claim, plus two tests that still could not fail. All three reproduced. Namespace: I reported "delete namespace restoration from both bridges -> 2 fail" but had only ablated the STREAMING path. The non-streaming path emits namespace separately (bridge.ts:1547), so deleting from both leaves the two sides equally degraded and a pure equality test stays green. Equality alone can never catch symmetric loss. The case now asserts `namespace === "ns"` absolutely on all three surfaces — snapshot, incremental frames and JSON — before comparing them. Deleting from both paths now fails. Function-kind discrimination: asserting the key set proved presence, not discrimination, because the generic named-tool fallback (parser.ts:194) recreates `fn` with identical marker state when the real function branch is deleted. The fixture now declares a non-empty schema and asserts it survives, since only the function branch carries parameters through. tool_search_call history: the suite only proved that definitions from `tool_search_output` were loaded, never that the search CALL survived in assistant history. Disabling the parser branch left everything green. If the call disappears, the next upstream request carries an orphaned result and providers reject the turn. Now asserts the call survives with its id and that the paired result carries the same id. Ablating the branch fails it. Also removed the last overclaim from the file header: it said the suite "pins where the risk actually lives", which read as completeness. It now says this is a starting slice and names what is absent. Verification: bun x tsc --noEmit clean; 159 pass / 0 fail across the 6 tool suites; privacy:scan passed; three new ablations confirmed red. --- tests/responses-tool-conformance.test.ts | 58 +++++++++++++++++++++--- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/tests/responses-tool-conformance.test.ts b/tests/responses-tool-conformance.test.ts index 7d68086a1..fcfe454f1 100644 --- a/tests/responses-tool-conformance.test.ts +++ b/tests/responses-tool-conformance.test.ts @@ -10,9 +10,12 @@ import { jsonItemTypes, jsonToolItems, streamedView } from "./helpers/responses- * The plan's premise was that a translator reading only top-level `tools` can silently erase * terminal, custom and namespace tools, and that the model then emits ordinary text and * completes normally — a failure that looks like model behavior rather than protocol loss. - * This suite pins where that risk actually lives in THIS codebase, which is not where the - * plan assumed: `additional_tools` is handled, so the residual exposure is malformed input, - * unknown tool kinds, and stream/non-stream divergence. + * That premise is partly obsolete here: `additional_tools` IS parsed and merged. This suite + * covers PART of the residual exposure — malformed input, unknown tool kinds, and + * stream/non-stream divergence. It is a STARTING slice of the 030-034 programme, not a + * complete conformance layer: the end-to-end declaration/call/result/second-turn execution + * loop, compaction and resume with a discovered tool, the full collision matrix, per-adapter + * declaration comparison, transport-error parity, and the conformance artifacts are absent. * * Several cases below deliberately pin CURRENT degradation rather than desired behavior. * They are labeled as such, so a future change to that behavior fails here loudly instead of @@ -99,7 +102,7 @@ describe("Responses Lite additional_tools declaration merge", () => { describe("Responses tool-kind discrimination", () => { it("maps each known kind onto its internal marker", () => { const parsed = parseRequest(request([], [ - { type: "function", name: "fn", parameters: { type: "object", properties: {} } }, + { type: "function", name: "fn", parameters: { type: "object", properties: { path: { type: "string" } } } }, { type: "custom", name: "freeform" }, { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, { type: "namespace", name: "ns", tools: [{ type: "function", name: "child", parameters: { type: "object", properties: {} } }] }, @@ -109,6 +112,10 @@ describe("Responses tool-kind discrimination", () => { // so the marker assertion alone would survive deleting the function branch. expect([...byName.keys()].sort()).toEqual(["child", "fn", "freeform", "tool_search"]); expect(byName.get("fn")?.freeform).toBeUndefined(); + // Presence is still not discrimination: the generic named-tool fallback would recreate + // `fn` with the same marker state. The declared SCHEMA only survives the real function + // branch, so assert it rather than the tool's mere existence. + expect(byName.get("fn")?.parameters).toEqual({ type: "object", properties: { path: { type: "string" } } } as never); expect(byName.get("freeform")?.freeform).toBe(true); expect(byName.get("tool_search")?.toolSearch).toBe(true); expect(byName.get("child")?.namespace).toBe("ns"); @@ -171,6 +178,38 @@ describe("tool_search call and output history", () => { expect(loaded?.loadedFromToolSearch).toBe(true); }); + it("preserves the search call itself in assistant history, paired with its result", () => { + // Loading the discovered DEFINITIONS is not enough: if the tool_search_call disappears + // from history, the next upstream request has an orphaned result and providers reject + // the turn. Asserts the call survives with its id, and that the paired result carries + // the same id so the two can be matched. + const parsed = parseRequest(request([ + { type: "message", role: "user", content: [{ type: "input_text", text: "find it" }] }, + { type: "tool_search_call", id: "ts_1", call_id: "ts_1", execution: "client", arguments: "{\"query\":\"repl\"}", status: "completed" }, + { + type: "tool_search_output", + call_id: "ts_1", + status: "completed", + execution: "client", + tools: [{ type: "function", name: "node_repl", parameters: { type: "object", properties: {} } }], + }, + ], [ + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + ])); + + const messages = parsed.context.messages; + const assistant = messages.find(message => message.role === "assistant"); + const call = Array.isArray(assistant?.content) + ? assistant.content.find(part => (part as { type?: string }).type === "toolCall") as { id?: string; name?: string } | undefined + : undefined; + expect(call?.name).toBe("tool_search"); + expect(call?.id).toBe("ts_1"); + + const result = messages.find(message => message.role === "toolResult") as { toolCallId?: string; toolName?: string } | undefined; + expect(result?.toolCallId).toBe("ts_1"); + expect(result?.toolName).toBe("tool_search"); + }); + it("flattens a namespaced tool discovered through tool_search", () => { const parsed = parseRequest(request([ { @@ -279,8 +318,15 @@ describe("streaming and non-streaming tool parity", () => { const nsCase = cases.find(entry => entry.label.startsWith("namespaced"))!; const view = await streamedView(nsCase.events, MODEL, nsMap, freeform, toolSearch); const json = jsonToolItems(nsCase.events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); - expect(view.snapshot[0]?.name).toBe("child"); - expect(json[0]?.name).toBe("child"); + + // ABSOLUTE assertions first. Equality alone is satisfied by EQUAL DEGRADATION: deleting + // namespace restoration from BOTH bridges keeps the two sides identical, so a pure + // comparison test would stay green while the identity was lost on the wire. + for (const item of [view.snapshot[0], view.incremental[0], json[0]]) { + expect(item?.name).toBe("child"); + expect(item?.namespace).toBe("ns"); + } + expect(view.incremental).toEqual(view.snapshot); expect(view.snapshot).toEqual(json); }); From f4cee1f4beaae6a76d9edef59caec2e815706fb3 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Thu, 13 Aug 2026 22:58:22 +0900 Subject: [PATCH 4/4] test(responses): stop claiming a discrimination the parser cannot express MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-audit was right a second time, and my previous commit message was wrong to list this as closed. The explicit `type === "function"` branch (parser.ts:157) and the generic named-tool fallback (parser.ts:194) both call `pushFn(t)`. For a NAMED tool they are observably identical, so no assertion can distinguish them — including the schema assertion I added, which the fallback preserves just as well. Deleting the explicit branch leaves the suite green, and that is a property of the code, not a hole in the test. Rather than invent an artificial discriminator, the comment now states the limitation plainly: this pins that a declared schema reaches the model intact, NOT which branch produced it, and it stays green when the explicit branch is deleted on purpose. That is the honest resolution. Claiming branch coverage here would have been the same failure this suite exists to prevent, one layer up. Verification: bun x tsc --noEmit clean; 23 pass / 0 fail; privacy:scan passed. --- tests/responses-tool-conformance.test.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/responses-tool-conformance.test.ts b/tests/responses-tool-conformance.test.ts index fcfe454f1..95bf30e55 100644 --- a/tests/responses-tool-conformance.test.ts +++ b/tests/responses-tool-conformance.test.ts @@ -108,13 +108,11 @@ describe("Responses tool-kind discrimination", () => { { type: "namespace", name: "ns", tools: [{ type: "function", name: "child", parameters: { type: "object", properties: {} } }] }, ])); const byName = new Map((parsed.context.tools ?? []).map(tool => [tool.name, tool])); - // Presence first: `byName.get("fn")?.freeform` is also undefined when "fn" is MISSING, - // so the marker assertion alone would survive deleting the function branch. - expect([...byName.keys()].sort()).toEqual(["child", "fn", "freeform", "tool_search"]); - expect(byName.get("fn")?.freeform).toBeUndefined(); - // Presence is still not discrimination: the generic named-tool fallback would recreate - // `fn` with the same marker state. The declared SCHEMA only survives the real function - // branch, so assert it rather than the tool's mere existence. + // NOTE: this asserts function-declaration BEHAVIOR, not branch discrimination. The + // explicit `type === "function"` branch and the generic named-tool fallback both call + // pushFn(t) (parser.ts:157 and :194), so they are observably identical for a named tool + // and NO assertion can tell them apart. Deleting the explicit branch keeps this green on + // purpose; what it does pin is that a declared schema reaches the model intact. expect(byName.get("fn")?.parameters).toEqual({ type: "object", properties: { path: { type: "string" } } } as never); expect(byName.get("freeform")?.freeform).toBe(true); expect(byName.get("tool_search")?.toolSearch).toBe(true);