From 94a323674b7e1e95fac0ded4f096523a89f17d4c Mon Sep 17 00:00:00 2001 From: Tajudeen Date: Wed, 22 Jul 2026 18:38:57 +0100 Subject: [PATCH 1/2] fix(cortexide): Ollama vision images + onboarding model name layout Pass image_url parts through to Ollama's native images field instead of dropping them, and let long model names flex-wrap in the add-model form. Co-authored-by: Cursor --- .../browser/react/src/settings/Settings.tsx | 8 ++--- .../cortexide/common/providerToolFormat.ts | 34 +++++++++++++++++++ .../llmMessage/sendLLMMessage.impl.ts | 14 ++------ .../test/common/providerToolFormat.test.ts | 26 +++++++++++++- 4 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/vs/workbench/contrib/cortexide/browser/react/src/settings/Settings.tsx b/src/vs/workbench/contrib/cortexide/browser/react/src/settings/Settings.tsx index 972be0ad5055..fe226c00dc22 100644 --- a/src/vs/workbench/contrib/cortexide/browser/react/src/settings/Settings.tsx +++ b/src/vs/workbench/contrib/cortexide/browser/react/src/settings/Settings.tsx @@ -605,7 +605,7 @@ export const ModelDump = ({ filteredProviders }: { filteredProviders?: ProviderN ) : isAddModelOpen ? (
-
+ {/* Provider dropdown */} @@ -616,19 +616,19 @@ export const ModelDump = ({ filteredProviders }: { filteredProviders?: ProviderN getOptionDisplayName={(pn) => pn ? displayInfoOfProviderName(pn).title : 'Provider Name'} getOptionDropdownName={(pn) => pn ? displayInfoOfProviderName(pn).title : 'Provider Name'} getOptionsEqual={(a, b) => a === b} - className="max-w-32 mx-2 w-full resize-none bg-void-bg-1 text-void-fg-1 placeholder:text-void-fg-3 border border-void-border-2 focus:border-void-border-1 py-1 px-2 rounded" + className="shrink-0 max-w-[140px] w-full resize-none bg-void-bg-1 text-void-fg-1 placeholder:text-void-fg-3 border border-void-border-2 focus:border-void-border-1 py-1 px-2 rounded" arrowTouchesText={false} /> - {/* Model name input */} + {/* Model name input — flex so long gguf names wrap instead of horizontal scroll (onboarding #67) */} diff --git a/src/vs/workbench/contrib/cortexide/common/providerToolFormat.ts b/src/vs/workbench/contrib/cortexide/common/providerToolFormat.ts index bb6af34843d7..b50c652a9744 100644 --- a/src/vs/workbench/contrib/cortexide/common/providerToolFormat.ts +++ b/src/vs/workbench/contrib/cortexide/common/providerToolFormat.ts @@ -124,6 +124,40 @@ export const effectiveSpecialToolFormat = ( isLocalInference: boolean, ): SpecialToolFormat => isLocalInference ? undefined : specialToolFormat +/** Ollama /api/chat message — content is string; optional base64 images for vision models. */ +export type OllamaChatMessage = { role: string; content: string; images?: string[] } + +/** Strip data-URL prefix from base64 image payloads for Ollama's `images` field. */ +export const base64FromDataUrl = (url: string): string => + url.startsWith('data:') ? url.replace(/^data:[^;]+;base64,/, '') : url + +/** + * Flatten OpenAI-format messages for Ollama's native /api/chat. + * Text parts join into `content`; image_url parts become base64 `images` (vision models). + */ +export const convertOpenAIMessagesToOllamaChat = (messages: LLMChatMessage[]): OllamaChatMessage[] => + messages.map((m) => { + const c = (m as { content?: unknown }).content + let content = '' + const images: string[] = [] + if (typeof c === 'string') { + content = c + } else if (Array.isArray(c)) { + for (const part of c) { + if (typeof part === 'string') { content += part } + else if (part?.type === 'text') { content += part.text ?? '' } + else if (part?.type === 'image_url' && part.image_url?.url) { + images.push(base64FromDataUrl(part.image_url.url)) + } + } + } else { + content = c == null ? '' : String(c) + } + const out: OllamaChatMessage = { role: (m as { role: string }).role, content } + if (images.length) { out.images = images } + return out + }) + /** * The running accumulator for an OpenAI-compatible streaming chat response: text, reasoning, and the * single tool call (name / args-JSON-string / id) assembled across deltas. The OpenAI streaming diff --git a/src/vs/workbench/contrib/cortexide/electron-main/llmMessage/sendLLMMessage.impl.ts b/src/vs/workbench/contrib/cortexide/electron-main/llmMessage/sendLLMMessage.impl.ts index 25434ca4e7fa..1b2a8c2c7787 100644 --- a/src/vs/workbench/contrib/cortexide/electron-main/llmMessage/sendLLMMessage.impl.ts +++ b/src/vs/workbench/contrib/cortexide/electron-main/llmMessage/sendLLMMessage.impl.ts @@ -15,7 +15,7 @@ import { GoogleAuth } from 'google-auth-library' /* eslint-enable */ import { GeminiLLMChatMessage, LLMChatMessage, LLMFIMMessage, ModelListParams, OllamaModelResponse, OnError, OnFinalMessage, OnText, RawToolCallObj } from '../../common/sendLLMMessageTypes.js'; -import { rawToolCallObjOfParamsStr, buildRawToolCallObj, sanitizeOpenAIMessagesForEmptyContent, toOpenAICompatibleTool, accumulateOpenAIChatDelta, buildTypedToolProperties, extractToolCallFromNonStreamingChoice, reduceGeminiChunk, finalizeGeminiToolId, effectiveSpecialToolFormat } from '../../common/providerToolFormat.js'; +import { rawToolCallObjOfParamsStr, buildRawToolCallObj, sanitizeOpenAIMessagesForEmptyContent, toOpenAICompatibleTool, accumulateOpenAIChatDelta, buildTypedToolProperties, extractToolCallFromNonStreamingChoice, reduceGeminiChunk, finalizeGeminiToolId, effectiveSpecialToolFormat, convertOpenAIMessagesToOllamaChat } from '../../common/providerToolFormat.js'; import { formatGeminiRateLimitError } from '../../common/providerErrorFormat.js'; import { ChatMode, displayInfoOfProviderName, FeatureName, ModelSelectionOptions, OverridesOfModel, ProviderName, SettingsOfProvider } from '../../common/cortexideSettingsTypes.js'; import { getSendableReasoningInfo, getModelCapabilities, getProviderCapabilities, defaultProviderSettings, getReservedOutputTokenSpace } from '../../common/modelCapabilities.js'; @@ -1206,17 +1206,7 @@ const sendOllamaChat = async ({ messages, onText, onFinalMessage, onError, setti } const messagesToSend = sanitizeOpenAIMessagesForEmptyContent(messages) - // Ollama's native /api/chat requires messages[].content to be a STRING, but OpenAI-format messages - // may carry array content (multimodal text/image parts). Flatten text parts to a string (images, - // rare for local coding, are dropped here — text-only agentic is the target). - const ollamaMessages = messagesToSend.map((m) => { - const c = (m as any).content - let content: string - if (typeof c === 'string') { content = c } - else if (Array.isArray(c)) { content = c.map((part: any) => typeof part === 'string' ? part : (part?.text ?? '')).join('') } - else { content = c == null ? '' : String(c) } - return { role: (m as any).role, content } - }) + const ollamaMessages = convertOpenAIMessagesToOllamaChat(messagesToSend) let fullTextSoFar = '' let firstTokenReceived = false diff --git a/src/vs/workbench/contrib/cortexide/test/common/providerToolFormat.test.ts b/src/vs/workbench/contrib/cortexide/test/common/providerToolFormat.test.ts index fe978c5fc5af..193e12a01923 100644 --- a/src/vs/workbench/contrib/cortexide/test/common/providerToolFormat.test.ts +++ b/src/vs/workbench/contrib/cortexide/test/common/providerToolFormat.test.ts @@ -5,7 +5,7 @@ import * as assert from 'assert'; import { suite, test } from 'mocha'; -import { buildRawToolCallObj, rawToolCallObjOfParamsStr, sanitizeOpenAIMessagesForEmptyContent as _sanitize, EMPTY_CONTENT_PLACEHOLDER, toOpenAICompatibleTool, accumulateOpenAIChatDelta, OpenAIChatAccumulator, OpenAIStreamDelta, effectiveSpecialToolFormat } from '../../common/providerToolFormat.js'; +import { rawToolCallObjOfParamsStr, buildRawToolCallObj, sanitizeOpenAIMessagesForEmptyContent as _sanitize, EMPTY_CONTENT_PLACEHOLDER, toOpenAICompatibleTool, accumulateOpenAIChatDelta, OpenAIChatAccumulator, OpenAIStreamDelta, effectiveSpecialToolFormat, convertOpenAIMessagesToOllamaChat, base64FromDataUrl } from '../../common/providerToolFormat.js'; import { LLMChatMessage } from '../../common/sendLLMMessageTypes.js'; import type { InternalToolInfo } from '../../common/prompt/prompts.js'; @@ -27,6 +27,30 @@ suite('effectiveSpecialToolFormat', () => { }); }); +suite('convertOpenAIMessagesToOllamaChat', () => { + test('string content passes through', () => { + assert.deepStrictEqual(convertOpenAIMessagesToOllamaChat([m({ role: 'user', content: 'hi' })]), [{ role: 'user', content: 'hi' }]); + }); + test('text parts flatten to content string', () => { + assert.deepStrictEqual( + convertOpenAIMessagesToOllamaChat([m({ role: 'user', content: [{ type: 'text', text: 'see ' }, { type: 'text', text: 'this' }] })]), + [{ role: 'user', content: 'see this' }], + ); + }); + test('image_url parts become base64 images array', () => { + const r = convertOpenAIMessagesToOllamaChat([m({ + role: 'user', + content: [{ type: 'text', text: 'what is this?' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,abc123' } }], + })]); + assert.strictEqual(r[0].content, 'what is this?'); + assert.deepStrictEqual(r[0].images, ['abc123']); + }); + test('base64FromDataUrl strips data-URL prefix', () => { + assert.strictEqual(base64FromDataUrl('data:image/jpeg;base64,XYZ'), 'XYZ'); + assert.strictEqual(base64FromDataUrl('rawb64'), 'rawb64'); + }); +}); + suite('buildRawToolCallObj', () => { test('object args -> RawToolCallObj with id/name/rawParams/doneParams/isDone', () => { assert.deepStrictEqual(buildRawToolCallObj('id1', 'read_file', { uri: '/a', line: '3' }), { From 58b58d497b7ebfdcf0000129043f6d0fa2dddd4d Mon Sep 17 00:00:00 2001 From: Tajudeen Date: Wed, 22 Jul 2026 18:50:45 +0100 Subject: [PATCH 2/2] ci(phase0-qa): pass GITHUB_TOKEN to preLaunch for CDP jobs Fixes 403 rate limits when fetching built-in extensions during Electron build on Windows/macOS workflow_dispatch runs. Co-authored-by: Cursor --- .github/workflows/phase0-qa.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/phase0-qa.yml b/.github/workflows/phase0-qa.yml index e04ecfd57ec3..0e77116e0858 100644 --- a/.github/workflows/phase0-qa.yml +++ b/.github/workflows/phase0-qa.yml @@ -77,6 +77,8 @@ jobs: - name: Build Electron dev app run: node build/lib/preLaunch.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run Phase 0 QA (unit + CDP) run: test/cortexide-smoke/run-phase0-qa.sh --cdp @@ -122,6 +124,8 @@ jobs: - name: Build Electron dev app run: node build/lib/preLaunch.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run Phase 0 QA (unit + CDP) shell: bash