From ebb3ae1ff209d660f540c21b5098cb04eab118f1 Mon Sep 17 00:00:00 2001 From: qer Date: Tue, 21 Jul 2026 18:11:06 +0800 Subject: [PATCH] feat(fetch-url): support fetching images as multimodal content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a URL points to an image (content-type: image/*), the fetcher now reads the binary body, base64-encodes it, and returns it as an UrlFetchResult with kind='image'. FetchURLTool detects this and emits a multimodal ContentPart[] (text + image_url) so the model can see the image directly instead of receiving an opaque binary blob description. Changes: - UrlFetchKind: add 'image' - UrlFetchResult: add optional imageData { mimeType, base64 } - LocalFetchURLProvider: detect image/* responses, read arrayBuffer, return base64-encoded image data - FetchURLTool: when kind === 'image', return ContentPart[] with image_url instead of plain text output - fetch-url.md: update description to mention image fetching - Tests: add coverage for image response handling in both provider and tool layers Fixes: Fetch 工具支持把图片也拉下来 --- .../src/tools/builtin/web/fetch-url.md | 2 +- .../src/tools/builtin/web/fetch-url.ts | 25 ++++++++++++-- .../src/tools/providers/local-fetch-url.ts | 21 +++++++++++- .../agent-core/test/tools/fetch-url.test.ts | 26 ++++++++++++++ .../tools/providers/local-fetch-url.test.ts | 34 +++++++++++++++++++ 5 files changed, 103 insertions(+), 5 deletions(-) diff --git a/packages/agent-core/src/tools/builtin/web/fetch-url.md b/packages/agent-core/src/tools/builtin/web/fetch-url.md index 30e98d6f9a..2cd235ad16 100644 --- a/packages/agent-core/src/tools/builtin/web/fetch-url.md +++ b/packages/agent-core/src/tools/builtin/web/fetch-url.md @@ -1,3 +1,3 @@ -Fetch content from a URL. The content is returned either as the main text extracted from the page, or as the full response body verbatim; a note at the top of the result states which of the two you received, so you can judge how complete it is. Use this when you need to read a specific web page. +Fetch content from a URL. The content is returned either as the main text extracted from the page, as the full response body verbatim, or (for image URLs) as the image itself; a note at the top of the result states which of the two you received, so you can judge how complete it is. Use this when you need to read a specific web page or fetch an image from the internet. Only fully-formed public `http`/`https` URLs are supported; other schemes and private or loopback addresses are not fetched. Very large pages may be truncated or refused. The fetch carries no login or session for the target site, so pages behind authentication (private repositories, internal dashboards) return a login page or an error instead of the real content — if the text you get back looks like a generic landing or sign-in page, treat that as the login wall, not the answer, and reach the content through a credentialed route (an authenticated CLI or MCP tool) instead. diff --git a/packages/agent-core/src/tools/builtin/web/fetch-url.ts b/packages/agent-core/src/tools/builtin/web/fetch-url.ts index d130e3977a..e983adc35c 100644 --- a/packages/agent-core/src/tools/builtin/web/fetch-url.ts +++ b/packages/agent-core/src/tools/builtin/web/fetch-url.ts @@ -25,14 +25,18 @@ import DESCRIPTION from './fetch-url.md?raw'; * returned verbatim, in full. * - `extracted` — the body was an HTML page; only the main article text * was extracted and returned. + * - `image` — the body was an image file; returned as a base64 data URL + * for direct model consumption. */ -export type UrlFetchKind = 'passthrough' | 'extracted'; +export type UrlFetchKind = 'passthrough' | 'extracted' | 'image'; export interface UrlFetchResult { - /** The text handed to the LLM. */ + /** The text handed to the LLM (for passthrough / extracted). */ content: string; /** Whether `content` is a verbatim passthrough or extracted main text. */ kind: UrlFetchKind; + /** Present when `kind === 'image'` — the image as a base64 data URL. */ + imageData?: { mimeType: string; base64: string }; } export interface UrlFetcher { @@ -89,7 +93,22 @@ export class FetchURLTool implements BuiltinTool { }: ExecutableToolContext, ): Promise { try { - const { content, kind } = await this.fetcher.fetch(args.url, { toolCallId }); + const { content, kind, imageData } = await this.fetcher.fetch(args.url, { toolCallId }); + + // Image response: return as multimodal ContentPart[] so the model + // can see the image directly. + if (kind === 'image' && imageData !== undefined) { + return { + output: [ + { type: 'text', text: `Fetched image from ${args.url}` }, + { + type: 'image_url', + imageUrl: { url: `data:${imageData.mimeType};base64,${imageData.base64}` }, + }, + ], + isError: false, + }; + } if (!content) { return { diff --git a/packages/agent-core/src/tools/providers/local-fetch-url.ts b/packages/agent-core/src/tools/providers/local-fetch-url.ts index 1b8e50844e..e2f9873173 100644 --- a/packages/agent-core/src/tools/providers/local-fetch-url.ts +++ b/packages/agent-core/src/tools/providers/local-fetch-url.ts @@ -237,6 +237,26 @@ export class LocalFetchURLProvider implements UrlFetcher { ); } + const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); + + // Image responses: read binary body and return as base64 for direct + // model consumption. + if (contentType.startsWith('image/')) { + const arrayBuffer = await response.arrayBuffer(); + const bytes = Buffer.from(arrayBuffer); + if (bytes.length > this.maxBytes) { + throw new Error( + `Response body too large: ${String(bytes.length)} bytes exceeds maxBytes (${String(this.maxBytes)}).`, + ); + } + const base64 = bytes.toString('base64'); + return { + content: `Image fetched (${contentType}, ${String(bytes.length)} bytes)`, + kind: 'image', + imageData: { mimeType: contentType.split(';')[0]!.trim(), base64 }, + }; + } + // Reject oversized responses before buffering the full body. const contentLengthRaw = response.headers.get('content-length'); if (contentLengthRaw !== null) { @@ -263,7 +283,6 @@ export class LocalFetchURLProvider implements UrlFetcher { ); } - const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); if (contentType.startsWith('text/plain') || contentType.startsWith('text/markdown')) { return { content: body, kind: 'passthrough' }; } diff --git a/packages/agent-core/test/tools/fetch-url.test.ts b/packages/agent-core/test/tools/fetch-url.test.ts index 06f319633e..aab86340e0 100644 --- a/packages/agent-core/test/tools/fetch-url.test.ts +++ b/packages/agent-core/test/tools/fetch-url.test.ts @@ -105,6 +105,32 @@ describe('FetchURLTool', () => { expect(out).toContain('# Raw markdown'); }); + it('returns image data as multimodal ContentPart[] when fetcher returns an image', async () => { + const fetcher: UrlFetcher = { + fetch: vi.fn().mockResolvedValue({ + content: 'Image fetched (image/png, 8 bytes)', + kind: 'image', + imageData: { mimeType: 'image/png', base64: 'dGVzdA==' }, + }), + }; + const tool = new FetchURLTool(fetcher); + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c-img', + args: { url: 'https://example.com/image.png' }, + signal, + }); + expect(result.isError).toBe(false); + expect(Array.isArray(result.output)).toBe(true); + const parts = result.output as Array<{ type: string }>; + expect(parts).toHaveLength(2); + expect(parts[0]).toMatchObject({ type: 'text' }); + expect(parts[1]).toMatchObject({ + type: 'image_url', + imageUrl: { url: 'data:image/png;base64,dGVzdA==' }, + }); + }); + it('returns empty message when fetcher returns empty string', async () => { const tool = new FetchURLTool(fakeFetcher('')); const result = await executeTool(tool, { diff --git a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts index ff23ea1d8d..979a2c68da 100644 --- a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts +++ b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts @@ -88,6 +88,40 @@ describe('LocalFetchURLProvider content kind', () => { expect(result.kind).toBe('extracted'); expect(result.content).toContain('quick brown fox'); }); + + it('reports image/* bodies as image kind with base64 data', async () => { + const pngBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const fetchImpl = vi.fn().mockResolvedValue( + new Response(pngBytes, { + status: 200, + headers: { 'content-type': 'image/png' }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + const result = await provider.fetch('https://example.com/image.png'); + + expect(result.kind).toBe('image'); + expect(result.imageData).toBeDefined(); + expect(result.imageData!.mimeType).toBe('image/png'); + expect(result.imageData!.base64).toBe(pngBytes.toString('base64')); + expect(result.content).toContain('Image fetched'); + }); + + it('rejects oversized image responses', async () => { + const bigBuffer = Buffer.alloc(11 * 1024 * 1024, 0xff); + const fetchImpl = vi.fn().mockResolvedValue( + new Response(bigBuffer, { + status: 200, + headers: { 'content-type': 'image/png' }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + await expect(provider.fetch('https://example.com/huge.png')).rejects.toThrow( + 'Response body too large', + ); + }); }); describe('LocalFetchURLProvider SSRF guard', () => {