[vscode-lm] Send attached images to the VS Code Language Model provider

.changeset/vscode-lm-image-input.md

@@ -0,0 +1,5 @@
+---
+"roo-cline": minor
+---
+
+Add image input support to the VS Code Language Model (vscode-lm) provider. Base64 image blocks are now sent as native `LanguageModelDataPart` image parts instead of text placeholders, and image capability is reported from the curated model family table. This raises the minimum supported VS Code version to 1.106.0, the first release where `LanguageModelDataPart` is available in the stable API.

apps/vscode-e2e/package.json

@@ -15,7 +15,7 @@
 		"@roo-code/types": "workspace:^",
 		"@types/mocha": "^10.0.10",
 		"@types/node": "20.x",
-		"@types/vscode": "^1.95.0",
+		"@types/vscode": "^1.120.0",
 		"@vscode/test-cli": "^0.0.11",
 		"@vscode/test-electron": "^2.4.0",
 		"glob": "^11.1.0",

packages/vscode-shim/src/__tests__/Additional.test.ts

@@ -7,6 +7,7 @@ import {
 	CodeActionKind,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,
@@ -261,6 +262,37 @@ describe("LanguageModelTextPart", () => {
 	})
 })
 
+describe("LanguageModelDataPart", () => {
+	const decode = (data: Uint8Array) => new TextDecoder().decode(data)
+
+	it("should create data part via constructor", () => {
+		const data = new Uint8Array([1, 2, 3])
+		const part = new LanguageModelDataPart(data, "image/png")
+
+		expect(part.data).toBe(data)
+		expect(part.mimeType).toBe("image/png")
+	})
+
+	describe("image()", () => {
+		it("should create image part preserving bytes and mime type", () => {
+			const data = new Uint8Array([137, 80, 78, 71])
+			const part = LanguageModelDataPart.image(data, "image/png")
+
+			expect(part).toBeInstanceOf(LanguageModelDataPart)
+			expect(part.data).toEqual(data)
+			expect(part.mimeType).toBe("image/png")
+		})
+
+		it("should round-trip base64-decoded image data", () => {
+			const base64 = Buffer.from("fake-image-bytes").toString("base64")
+			const part = LanguageModelDataPart.image(new Uint8Array(Buffer.from(base64, "base64")), "image/jpeg")
+
+			expect(decode(part.data)).toBe("fake-image-bytes")
+			expect(part.mimeType).toBe("image/jpeg")
+		})
+	})
+})
+
 describe("LanguageModelToolCallPart", () => {
 	it("should create tool call part", () => {
 		const part = new LanguageModelToolCallPart("call-123", "searchFiles", { query: "test" })

packages/vscode-shim/src/api/create-vscode-api-mock.ts

@@ -21,6 +21,7 @@ import {
 	CodeActionKind,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,
@@ -161,6 +162,7 @@ export function createVSCodeAPIMock(
 		CancellationTokenSource,
 		CodeLens,
 		LanguageModelTextPart,
+		LanguageModelDataPart,
 		LanguageModelToolCallPart,
 		LanguageModelToolResultPart,
 		ExtensionContext: ExtensionContextImpl,

packages/vscode-shim/src/classes/Additional.ts

@@ -112,6 +112,25 @@ export class LanguageModelTextPart {
 	constructor(public value: string) {}
 }
 
+export class LanguageModelDataPart {
+	constructor(
+		public data: Uint8Array,
+		public mimeType: string,
+	) {}
+
+	static image(data: Uint8Array, mimeType: string): LanguageModelDataPart {
+		return new LanguageModelDataPart(data, mimeType)
+	}
+
+	static text(value: string, mimeType: string = "text/plain"): LanguageModelDataPart {
+		return new LanguageModelDataPart(new TextEncoder().encode(value), mimeType)
+	}
+
+	static json(value: unknown, mimeType: string = "application/json"): LanguageModelDataPart {
+		return new LanguageModelDataPart(new TextEncoder().encode(JSON.stringify(value)), mimeType)
+	}
+}
+
 export class LanguageModelToolCallPart {
 	constructor(
 		public callId: string,

packages/vscode-shim/src/index.ts

@@ -30,6 +30,7 @@ export {
 	CancellationTokenSource,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,

packages/vscode-shim/src/vscode.ts

@@ -23,6 +23,7 @@ export {
 	CodeActionKind,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,

src/api/providers/__tests__/vscode-lm.spec.ts

@@ -497,6 +497,35 @@ describe("VsCodeLmHandler", () => {
 			const model = handler.getModel()
 			expect(model.info.contextWindow).toBe(openAiModelInfoSaneDefaults.contextWindow)
 		})
+
+		it("resolves supportsImages to true for a family the static table marks image-capable", async () => {
+			const mockModel = { ...mockLanguageModelChat, family: "claude-sonnet-4.5" }
+			;(vscode.lm.selectChatModels as Mock).mockResolvedValue([mockModel])
+			handler["client"] = null
+			await handler.initializeClient()
+
+			expect(vscodeLlmModels["claude-sonnet-4.5"].supportsImages).toBe(true)
+			expect(handler.getModel().info.supportsImages).toBe(true)
+		})
+
+		it("resolves supportsImages to false for a family the static table marks image-incapable", async () => {
+			const mockModel = { ...mockLanguageModelChat, family: "gpt-4o-mini" }
+			;(vscode.lm.selectChatModels as Mock).mockResolvedValue([mockModel])
+			handler["client"] = null
+			await handler.initializeClient()
+
+			expect(vscodeLlmModels["gpt-4o-mini"].supportsImages).toBe(false)
+			expect(handler.getModel().info.supportsImages).toBe(false)
+		})
+
+		it("defaults supportsImages to false for a family missing from the static table", async () => {
+			const mockModel = { ...mockLanguageModelChat, family: "totally-unknown-family" }
+			;(vscode.lm.selectChatModels as Mock).mockResolvedValue([mockModel])
+			handler["client"] = null
+			await handler.initializeClient()
+
+			expect(handler.getModel().info.supportsImages).toBe(false)
+		})
 	})
 
 	describe("countTokens", () => {

src/api/providers/vscode-lm.ts

@@ -886,7 +886,8 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan
 					typeof this.client.maxInputTokens === "number"
 						? Math.max(0, this.client.maxInputTokens)
 						: openAiModelInfoSaneDefaults.contextWindow,
-				supportsImages: false, // VSCode Language Model API currently doesn't support image inputs
+				supportsImages:
+					vscodeLlmModels[this.client.family as keyof typeof vscodeLlmModels]?.supportsImages ?? false,
 				supportsPromptCache: true,
 				inputPrice: 0,
 				outputPrice: 0,

src/api/transform/__tests__/vscode-lm-format.spec.ts

@@ -34,6 +34,11 @@ interface MockLanguageModelToolResultPart {
 	content: MockLanguageModelTextPart[]
 }
 
+interface MockLanguageModelDataPart {
+	data: Uint8Array
+	mimeType: string
+}
+
 // Mock vscode namespace
 vitest.mock("vscode", () => {
 	const LanguageModelChatMessageRole = {
@@ -55,6 +60,18 @@ vitest.mock("vscode", () => {
 		) {}
 	}
 
+	// The real vscode.LanguageModelDataPart carries no discriminator field, only data and mimeType.
+	class MockLanguageModelDataPart {
+		constructor(
+			public data: Uint8Array,
+			public mimeType: string,
+		) {}
+
+		static image(data: Uint8Array, mimeType: string) {
+			return new MockLanguageModelDataPart(data, mimeType)
+		}
+	}
+
 	class MockLanguageModelToolResultPart {
 		type = "tool_result"
 		constructor(
@@ -78,6 +95,7 @@ vitest.mock("vscode", () => {
 		},
 		LanguageModelChatMessageRole,
 		LanguageModelTextPart: MockLanguageModelTextPart,
+		LanguageModelDataPart: MockLanguageModelDataPart,
 		LanguageModelToolCallPart: MockLanguageModelToolCallPart,
 		LanguageModelToolResultPart: MockLanguageModelToolResultPart,
 	}
@@ -155,7 +173,8 @@ describe("convertToVsCodeLmMessages", () => {
 		expect(toolCall.type).toBe("tool_call")
 	})
 
-	it("should handle image blocks with appropriate placeholders", () => {
+	it("should convert base64 image blocks into data parts with decoded bytes", () => {
+		const pngBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
 		const messages: Anthropic.Messages.MessageParam[] = [
 			{
 				role: "user",
@@ -166,7 +185,7 @@ describe("convertToVsCodeLmMessages", () => {
 						source: {
 							type: "base64",
 							media_type: "image/png",
-							data: "base64data",
+							data: Buffer.from(pngBytes).toString("base64"),
 						},
 					},
 				],
@@ -176,8 +195,79 @@ describe("convertToVsCodeLmMessages", () => {
 		const result = convertToVsCodeLmMessages(messages)
 
 		expect(result).toHaveLength(1)
+		const imagePart = result[0].content[1] as unknown as MockLanguageModelDataPart
+		expect(imagePart.mimeType).toBe("image/png")
+		// Guards against passing the base64 string through instead of the decoded bytes.
+		expect(Array.from(imagePart.data)).toEqual(Array.from(pngBytes))
+	})
+
+	it("should keep a text placeholder for URL-sourced images", () => {
+		const messages: Anthropic.Messages.MessageParam[] = [
+			{
+				role: "user",
+				content: [
+					{ type: "text", text: "Look at this:" },
+					// The SDK's ImageBlockParam.Source only models base64 sources, but URL-sourced
+					// images do reach this transform at runtime, so exercise that branch directly.
+					{
+						type: "image",
+						source: {
+							type: "url",
+							url: "https://example.com/image.png",
+						},
+					} as unknown as Anthropic.ImageBlockParam,
+				],
+			},
+		]
+
+		const result = convertToVsCodeLmMessages(messages)
+
+		expect(result).toHaveLength(1)
+		const imagePlaceholder = result[0].content[1] as MockLanguageModelTextPart
+		expect(imagePlaceholder.value).toBe("[Image (url): unknown media-type not supported by VSCode LM API]")
+	})
+
+	it("should fall back to a text placeholder when base64 data decodes to no bytes", () => {
+		const messages: Anthropic.Messages.MessageParam[] = [
+			{
+				role: "user",
+				content: [
+					{ type: "text", text: "Look at this:" },
+					// Buffer.from strips non-base64 characters rather than throwing, so the data must
+					// contain NO base64 alphabet characters at all to actually decode to zero bytes.
+					{ type: "image", source: { type: "base64", media_type: "image/png", data: "!!!" } },
+				],
+			},
+		]
+
+		const result = convertToVsCodeLmMessages(messages)
+
+		const imagePlaceholder = result[0].content[1] as MockLanguageModelTextPart
+		expect(imagePlaceholder.value).toBe("[Image (base64): image/png not supported by VSCode LM API]")
+	})
+
+	it("should fall back to a text placeholder for an unsupported media type", () => {
+		const messages: Anthropic.Messages.MessageParam[] = [
+			{
+				role: "user",
+				content: [
+					{ type: "text", text: "Look at this:" },
+					{
+						type: "image",
+						source: {
+							type: "base64",
+							media_type: "image/bmp",
+							data: Buffer.from([1, 2, 3]).toString("base64"),
+						},
+					} as unknown as Anthropic.ImageBlockParam,
+				],
+			},
+		]
+
+		const result = convertToVsCodeLmMessages(messages)
+
 		const imagePlaceholder = result[0].content[1] as MockLanguageModelTextPart
-		expect(imagePlaceholder.value).toContain("[Image (base64): image/png not supported by VSCode LM API]")
+		expect(imagePlaceholder.value).toBe("[Image (base64): image/bmp not supported by VSCode LM API]")
 	})
 
 	it("should replace an unpaired surrogate in a tool_result with U+FFFD", () => {

src/api/transform/vscode-lm-format.ts

@@ -45,6 +45,31 @@ export function sanitizeSurrogates(text: string): string {
 	return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "\uFFFD")
 }
 
+const SUPPORTED_IMAGE_MEDIA_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"])
+
+function convertImagePart(part: {
+	source?: { type?: string; media_type?: string; data?: string }
+}): vscode.LanguageModelTextPart | vscode.LanguageModelDataPart {
+	if (
+		part.source?.type === "base64" &&
+		part.source.data &&
+		part.source.media_type &&
+		SUPPORTED_IMAGE_MEDIA_TYPES.has(part.source.media_type)
+	) {
+		// MUST be raw decoded bytes: passing the base64 string re-encodes it and corrupts the image.
+		const bytes = new Uint8Array(Buffer.from(part.source.data, "base64"))
+		// Buffer.from never throws on malformed base64 — it yields empty/garbage bytes — so an empty
+		// result is the only signal that decoding failed, and it falls through to a visible placeholder.
+		if (bytes.length > 0) {
+			return vscode.LanguageModelDataPart.image(bytes, part.source.media_type)
+		}
+	}
+
+	return new vscode.LanguageModelTextPart(
+		`[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`,
+	)
+}
+
 export function convertToVsCodeLmMessages(
 	anthropicMessages: Anthropic.Messages.MessageParam[],
 ): vscode.LanguageModelChatMessage[] {
@@ -85,14 +110,12 @@ export function convertToVsCodeLmMessages(
 					// Convert tool messages to ToolResultParts
 					...toolMessages.map((toolMessage) => {
 						// Process tool result content into TextParts
-						const toolContentParts: vscode.LanguageModelTextPart[] =
+						const toolContentParts: (vscode.LanguageModelTextPart | vscode.LanguageModelDataPart)[] =
 							typeof toolMessage.content === "string"
 								? [new vscode.LanguageModelTextPart(sanitizeSurrogates(toolMessage.content))]
 								: (toolMessage.content?.map((part) => {
 										if (part.type === "image") {
-											return new vscode.LanguageModelTextPart(
-												`[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`,
-											)
+											return convertImagePart(part)
 										}
 										return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
 									}) ?? [new vscode.LanguageModelTextPart("")])
@@ -103,9 +126,7 @@ export function convertToVsCodeLmMessages(
 					// Convert non-tool messages to TextParts after tool messages
 					...nonToolMessages.map((part) => {
 						if (part.type === "image") {
-							return new vscode.LanguageModelTextPart(
-								`[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`,
-							)
+							return convertImagePart(part)
 						}
 						return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
 					}),

src/package.json

@@ -10,7 +10,7 @@
 		"theme": "dark"
 	},
 	"engines": {
-		"vscode": "^1.84.0",
+		"vscode": "^1.106.0",
 		"node": "20.19.2"
 	},
 	"author": {
@@ -546,7 +546,7 @@
 		"@types/string-similarity": "^4.0.2",
 		"@types/tmp": "^0.2.6",
 		"@types/turndown": "^5.0.5",
-		"@types/vscode": "^1.84.0",
+		"@types/vscode": "^1.120.0",
 		"@vscode/test-electron": "^2.5.2",
 		"@vscode/vsce": "3.3.2",
 		"ai": "^6.0.75",