From fa3e542eae889d797d1d4e5fd033785c19dbffa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20LIARD?= Date: Tue, 28 Jul 2026 22:59:48 +0200 Subject: [PATCH] fix(openai): preserve tool-result images on the Responses (Codex) path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tool that returns an image (screenshot, computer-use, an MCP vision tool) puts an `image` block in its `tool_result`. On the Responses path grob built the `function_call_output` with `content.to_string()`, and `ToolResultContent`'s Display renders an image block as the literal text `[Image]`. The model received no pixels and hallucinated — verified live through grob against gpt-5.5: before: tool image → "[Image]" → model answered "White" after: tool image → input_image part → model answered "Magenta" The Codex backend accepts `function_call_output.output` as an array of `input_text`/`input_image` parts (probed directly); text-only results stay a bare string so the common path and prompt cache are unchanged. `output` widened from `String` to `serde_json::Value` to carry either shape. Unknown tool-result blocks are now folded into text instead of being dropped. Same class as the message-content vision fix; this closes the tool-output half. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/providers/openai/transform.rs | 100 +++++++++++++++++++++++++++++- src/providers/openai/types.rs | 10 ++- 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/src/providers/openai/transform.rs b/src/providers/openai/transform.rs index 00fca99..17210a1 100644 --- a/src/providers/openai/transform.rs +++ b/src/providers/openai/transform.rs @@ -1305,7 +1305,7 @@ fn push_blocks_as_items( flush_message_parts(items, role, &mut text, &mut parts); items.push(OpenAIResponsesItem::FunctionCallOutput { call_id: tool_use_id.clone(), - output: content.to_string(), + output: responses_tool_output(content), }); } _ => {} @@ -1359,6 +1359,63 @@ fn responses_message_content(role: &str, text: String) -> serde_json::Value { serde_json::Value::Array(vec![responses_text_part(role, text)]) } +/// Serializes a tool result as a Responses `function_call_output.output`. +/// +/// Text-only results stay a bare JSON string (unchanged wire shape, so the +/// prompt cache is undisturbed). When the tool returned an image, the output +/// becomes an array of `input_text`/`input_image` parts in original order — the +/// Codex backend accepts this and the model sees the actual pixels, instead of +/// the `[Image]` placeholder `Display` produced, which made it hallucinate. +fn responses_tool_output(content: &crate::models::ToolResultContent) -> serde_json::Value { + use crate::models::{KnownToolResultBlock, ToolResultBlock, ToolResultContent}; + + let ToolResultContent::Blocks(blocks) = content else { + return serde_json::Value::String(content.to_string()); + }; + let has_image = blocks.iter().any(|b| { + matches!( + b, + ToolResultBlock::Known(KnownToolResultBlock::Image { .. }) + ) + }); + if !has_image { + return serde_json::Value::String(content.to_string()); + } + + let mut parts: Vec = Vec::new(); + let mut text = String::new(); + let flush_text = |text: &mut String, parts: &mut Vec| { + if !text.is_empty() { + parts.push(serde_json::json!({ "type": "input_text", "text": std::mem::take(text) })); + } + }; + for block in blocks { + match block { + ToolResultBlock::Known(KnownToolResultBlock::Text { text: t }) => { + if !text.is_empty() { + text.push('\n'); + } + text.push_str(t); + } + ToolResultBlock::Known(KnownToolResultBlock::Image { source }) => { + flush_text(&mut text, &mut parts); + if let Some(part) = responses_image_part(source) { + parts.push(part); + } + } + // Preserve unknown blocks as text rather than dropping them silently. + ToolResultBlock::Unknown(v) => { + if !text.is_empty() { + text.push('\n'); + } + text.push_str(&v.to_string()); + } + } + } + flush_text(&mut text, &mut parts); + serde_json::Value::Array(parts) +} + /// Builds a Responses-API `input_image` part from an Anthropic image block. /// /// Base64 sources become a `data:` URI; URL sources pass through. Returns `None` @@ -1896,6 +1953,47 @@ mod tests { assert_eq!(part["image_url"], "https://example.com/cat.png"); } + #[test] + fn tool_result_text_stays_a_bare_string() { + use crate::models::ToolResultContent; + // Text-only results must keep the plain-string wire shape (cache-friendly). + let out = responses_tool_output(&ToolResultContent::Text("file1\nfile2".to_string())); + assert_eq!(out, serde_json::json!("file1\nfile2")); + } + + #[test] + fn tool_result_image_survives_as_input_image() { + use crate::models::{ + ImageSource, KnownToolResultBlock, ToolResultBlock, ToolResultContent, + }; + // A tool that returns an image: the image must reach the model as pixels, + // not the literal "[Image]" text `Display` produced. + let content = ToolResultContent::Blocks(vec![ + ToolResultBlock::Known(KnownToolResultBlock::Text { + text: "screenshot:".to_string(), + }), + ToolResultBlock::Known(KnownToolResultBlock::Image { + source: ImageSource { + r#type: "base64".to_string(), + media_type: Some("image/png".to_string()), + data: Some("aGVsbG8=".to_string()), + url: None, + }, + }), + ]); + + let out = responses_tool_output(&content); + let parts = out + .as_array() + .expect("image output must be an array of parts"); + assert_eq!(parts[0]["type"], "input_text"); + assert_eq!(parts[0]["text"], "screenshot:"); + assert_eq!(parts[1]["type"], "input_image"); + assert_eq!(parts[1]["image_url"], "data:image/png;base64,aGVsbG8="); + // No "[Image]" placeholder anywhere. + assert!(!out.to_string().contains("[Image]")); + } + #[test] fn responses_request_without_tools_keeps_full_instructions() { let mut request = base_request(); diff --git a/src/providers/openai/types.rs b/src/providers/openai/types.rs index 9ba9e65..4d8dda3 100644 --- a/src/providers/openai/types.rs +++ b/src/providers/openai/types.rs @@ -122,7 +122,15 @@ pub(crate) enum OpenAIResponsesItem { arguments: String, }, /// The output of a tool call, fed back to the model. - FunctionCallOutput { call_id: String, output: String }, + /// + /// `output` is a bare string for text results (cache-friendly) or an array + /// of `input_text`/`input_image` parts when the tool returned an image, so a + /// screenshot/vision tool's output reaches the model as pixels, not the + /// literal text `"[Image]"`. + FunctionCallOutput { + call_id: String, + output: serde_json::Value, + }, /// A `reasoning` item replayed verbatim from a previous turn. /// /// Carries the backend's opaque `encrypted_content`, so it is forwarded