From 1e75c7f7ce04271bfc38c94a54bdf3be0b258a88 Mon Sep 17 00:00:00 2001 From: Tajudeen Date: Thu, 23 Jul 2026 14:31:54 +0100 Subject: [PATCH] refactor(ui): extract submit vision validation from SidebarChat Move attachment conversion and vision checks into prepareChatSubmit and useVisionAttachmentHandlers so SidebarChat focuses on layout orchestration. Co-authored-by: Cursor --- package.json | 2 +- .../react/src/sidebar-tsx/SidebarChat.tsx | 182 ++---------------- .../sidebar-tsx/composer/prepareChatSubmit.ts | 102 ++++++++++ .../composer/useVisionAttachmentHandlers.ts | 92 +++++++++ .../common/prepareChatAttachments.ts | 61 ++++++ .../common/prepareChatAttachments.test.ts | 42 ++++ 6 files changed, 317 insertions(+), 164 deletions(-) create mode 100644 src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/prepareChatSubmit.ts create mode 100644 src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/useVisionAttachmentHandlers.ts create mode 100644 src/vs/workbench/contrib/cortexide/common/prepareChatAttachments.ts create mode 100644 src/vs/workbench/contrib/cortexide/test/common/prepareChatAttachments.test.ts diff --git a/package.json b/package.json index f252422e3df..75c923431fb 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "test-browser": "npx playwright install && node test/unit/browser/index.js", "test-browser-no-install": "node test/unit/browser/index.js", "test-node": "mocha test/unit/node/index.js --delay --ui=tdd --timeout=5000 --exit", - "test-phase0-qa": "npm run test-node -- --runGlob \"**/cortexide/test/common/{providerSettingsValidation,onboardingHelpers,attachFileToChat,chatThreadStorageReviver,providerToolFormat,phase0ClaimVerification,modelCapabilities,menubarStackingFix,designSystem,atReferenceTokens,resolveAtReferences}.test.js\"", + "test-phase0-qa": "npm run test-node -- --runGlob \"**/cortexide/test/common/{providerSettingsValidation,onboardingHelpers,attachFileToChat,chatThreadStorageReviver,providerToolFormat,phase0ClaimVerification,modelCapabilities,menubarStackingFix,designSystem,atReferenceTokens,resolveAtReferences,prepareChatAttachments}.test.js\"", "test-extension": "vscode-test", "test-build-scripts": "cd build && npm run test", "check-cyclic-dependencies": "node build/lib/checkCyclicDependencies.ts out", diff --git a/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/SidebarChat.tsx b/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/SidebarChat.tsx index 0a8f9b825d5..ba8f6f63999 100644 --- a/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/SidebarChat.tsx +++ b/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/SidebarChat.tsx @@ -19,12 +19,14 @@ import { StagingContextChips } from './composer/StagingContextChips.js'; import { useContextUsage } from './composer/useContextUsage.js'; import { resolveAtReferencesInMessage } from '../../../../common/resolveAtReferences.js'; import { handleSlashCommand } from './composer/handleSlashCommand.js'; +import { prepareChatSubmitAttachments } from './composer/prepareChatSubmit.js'; +import { useVisionAttachmentHandlers } from './composer/useVisionAttachmentHandlers.js'; import { LandingPage } from './landing/LandingPage.js'; import { ComposerTabs } from './chrome/ComposerTabs.js'; import { ThreadHeader } from './chrome/ThreadHeader.js'; import { CORTEXIDE_OPEN_SETTINGS_ACTION_ID } from '../../../cortexideSettingsPane.js'; import { isFeatureNameDisabled } from '../../../../../../../workbench/contrib/cortexide/common/cortexideSettingsTypes.js'; -import { StagingSelectionItem, ChatImageAttachment, ChatPDFAttachment } from '../../../../common/chatThreadServiceTypes.js'; +import { StagingSelectionItem } from '../../../../common/chatThreadServiceTypes.js'; import ErrorBoundary from './ErrorBoundary.js'; import { useImageAttachments } from '../util/useImageAttachments.js'; import { usePDFAttachments } from '../util/usePDFAttachments.js'; @@ -108,83 +110,13 @@ export const SidebarChat = () => { validationError: pdfValidationError, } = usePDFAttachments(); - // Wrapper to check vision capabilities before adding PDFs - // PDFs are more forgiving than images - they can work with non-vision models via text extraction - const addPDFs = useCallback(async (files: File[]) => { - const currentModelSel = settingsState.modelSelectionOfFeature['Chat']; - - // In auto mode, skip vision capability check - the router will select an appropriate model - // PDFs can also work with non-vision models via text extraction, so we're more lenient - if (currentModelSel?.providerName === 'auto' && currentModelSel?.modelName === 'auto') { - await addPDFsRaw(files); - return; - } - - // For non-auto mode, allow PDFs even without vision models (they can use text extraction) - // But we could optionally warn if no vision models are available - await addPDFsRaw(files); - }, [addPDFsRaw, settingsState]); - - // Wrapper to check vision capabilities before adding images - const addImages = useCallback(async (files: File[]) => { - const currentModelSel = settingsState.modelSelectionOfFeature['Chat']; - - // In auto mode, skip vision capability check - the router will select an appropriate model - if (currentModelSel?.providerName === 'auto' && currentModelSel?.modelName === 'auto') { - await addImagesRaw(files); - return; - } - - // Check if user has vision-capable API keys or Ollama vision models - const { isSelectedModelVisionCapable, checkOllamaModelVisionCapable, hasVisionCapableApiKey, hasOllamaVisionModel, isOllamaAccessible } = await import('../util/visionModelHelper.js'); - - // First, check if the currently selected model is vision-capable (synchronous check) - let selectedIsVision = isSelectedModelVisionCapable(currentModelSel, settingsState.settingsOfProvider); - - // If Ollama model and not detected by name, query Ollama API directly (async) - if (!selectedIsVision && currentModelSel?.providerName === 'ollama') { - const ollamaAccessible = await isOllamaAccessible(); - if (ollamaAccessible) { - selectedIsVision = await checkOllamaModelVisionCapable(currentModelSel.modelName); - } - } - - if (selectedIsVision) { - // User has selected a vision-capable model, proceed - await addImagesRaw(files); - return; - } - - // If not selected, check if they have any vision-capable options available - const hasApiKey = hasVisionCapableApiKey(settingsState.settingsOfProvider, currentModelSel); - const ollamaAccessible = await isOllamaAccessible(); - const hasOllamaVision = ollamaAccessible && await hasOllamaVisionModel(); - - if (!hasApiKey && !hasOllamaVision) { - // Show notification with option to open Ollama setup - const notificationService = accessor.get('INotificationService'); - const commandService = accessor.get('ICommandService'); - - notificationService.notify({ - severity: 2, // Severity.Warning - message: 'No vision-capable models available. Please set up an API key (Anthropic, OpenAI, or Gemini) or install an Ollama vision model (e.g., llava, bakllava).', - actions: { - primary: [{ - id: 'void.vision.setup', - label: 'Setup Ollama Vision Models', - tooltip: '', - class: undefined, - enabled: true, - run: () => commandService.executeCommand(CORTEXIDE_OPEN_SETTINGS_ACTION_ID), - }], - }, - }); - return; - } - - // User has vision support available but not selected, proceed anyway (they might switch models) - await addImagesRaw(files); - }, [addImagesRaw, settingsState, accessor]); + const { addImages, addPDFs } = useVisionAttachmentHandlers({ + settingsState, + accessor, + settingsCommandId: CORTEXIDE_OPEN_SETTINGS_ACTION_ID, + addImagesRaw, + addPDFsRaw, + }); // Compute isDisabled - ensure it's reactive to settings changes const isDisabled = useMemo(() => { @@ -237,92 +169,16 @@ export const SidebarChat = () => { notificationService, }); - // Convert image attachments to ChatImageAttachment format - const images: ChatImageAttachment[] = imageAttachments - .filter(att => att.uploadStatus === 'success' || !att.uploadStatus) - .map(att => ({ - id: att.id, - data: att.data, - mimeType: att.mimeType, - filename: att.filename, - width: att.width, - height: att.height, - size: att.size, - })); - - // Check if any PDFs are still processing - const processingPDFs = pdfAttachments.filter( - att => att.uploadStatus === 'uploading' || att.uploadStatus === 'processing' - ); - - if (processingPDFs.length > 0) { - const processingNames = processingPDFs.map(p => p.filename).join(', '); - notificationService.warn(`Some PDFs are still processing: ${processingNames}. They will be sent but may not have extracted text available yet.`); - } + const prepared = await prepareChatSubmitAttachments({ + imageAttachments, + pdfAttachments, + modelSelection: settingsState.modelSelectionOfFeature['Chat'], + settingsOfProvider: settingsState.settingsOfProvider, + notificationService, + }); + if (!prepared.ok) return; - // Convert PDF attachments to ChatPDFAttachment format - // Include PDFs that are successful, have no status, or are still processing (they might have partial data) - // Exclude only failed PDFs - const pdfs: ChatPDFAttachment[] = pdfAttachments - .filter(att => att.uploadStatus !== 'failed') - .map(att => ({ - id: att.id, - data: att.data, - filename: att.filename, - size: att.size, - pageCount: att.pageCount, - selectedPages: att.selectedPages, - extractedText: att.extractedText, - pagePreviews: att.pagePreviews, - })); - - // Validate that model supports vision/PDFs if attachments are present - const currentModelSel = settingsState.modelSelectionOfFeature['Chat']; - if ((images.length > 0 || pdfs.length > 0) && currentModelSel) { - const { isSelectedModelVisionCapable, checkOllamaModelVisionCapable, hasVisionCapableApiKey, hasOllamaVisionModel, isOllamaAccessible } = await import('../util/visionModelHelper.js'); - - // In auto mode, check if user has any vision-capable models available - if (currentModelSel.providerName === 'auto' && currentModelSel.modelName === 'auto') { - // Images require vision-capable models (no fallback) - if (images.length > 0) { - const hasApiKey = hasVisionCapableApiKey(settingsState.settingsOfProvider, currentModelSel); - const ollamaAccessible = await isOllamaAccessible(); - const hasOllamaVision = ollamaAccessible && await hasOllamaVisionModel(); - - if (!hasApiKey && !hasOllamaVision) { - notificationService.error('No vision-capable models available. Please set up an API key (Anthropic, OpenAI, or Gemini) or install an Ollama vision model (e.g., llava, bakllava) to use images.'); - return; - } - } - // PDFs can work with non-vision models via text extraction, so we allow them even without vision-capable models - // If vision-capable models are available, router will select appropriate model - } else { - // For non-auto mode, check if the selected model is vision-capable - let isVisionCapable = isSelectedModelVisionCapable(currentModelSel, settingsState.settingsOfProvider); - - // If Ollama, check via API - if (!isVisionCapable && currentModelSel.providerName === 'ollama') { - const ollamaAccessible = await isOllamaAccessible(); - if (ollamaAccessible) { - isVisionCapable = await checkOllamaModelVisionCapable(currentModelSel.modelName); - } - } - - // If not vision-capable, show error - if (!isVisionCapable) { - const hasApiKey = hasVisionCapableApiKey(settingsState.settingsOfProvider, currentModelSel); - const ollamaAccessible = await isOllamaAccessible(); - const hasOllamaVision = ollamaAccessible && await hasOllamaVisionModel(); - - if (!hasApiKey && !hasOllamaVision) { - notificationService.error('The selected model does not support images or PDFs. Please select a vision-capable model (e.g., Claude, GPT-4, Gemini, or an Ollama vision model like llava).'); - return; - } else { - notificationService.warn('The selected model may not support images or PDFs. Consider switching to a vision-capable model for better results.'); - } - } - } - } + const { images, pdfs } = prepared; // Capture staging selections BEFORE clearing them, so they're included in the message const stagingSelections = chatThreadsState.allThreads[currentThread.id]?.state?.stagingSelections || [] diff --git a/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/prepareChatSubmit.ts b/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/prepareChatSubmit.ts new file mode 100644 index 00000000000..162b73e051e --- /dev/null +++ b/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/prepareChatSubmit.ts @@ -0,0 +1,102 @@ +/*-------------------------------------------------------------------------------------- + * Copyright 2025 Glass Devtools, Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. + *--------------------------------------------------------------------------------------*/ + +import { ChatImageAttachment, ChatPDFAttachment } from '../../../../common/chatThreadServiceTypes.js'; +import { + getProcessingPDFFilenames, + ImageAttachmentDraft, + PDFAttachmentDraft, + toChatImageAttachments, + toChatPDFAttachments, +} from '../../../../common/prepareChatAttachments.js'; +import { ModelSelection, SettingsOfProvider } from '../../../../common/cortexideSettingsTypes.js'; + +type NotificationServiceLike = { + warn: (message: string) => void; + error: (message: string) => void; +}; + +export type PrepareChatSubmitParams = { + imageAttachments: ImageAttachmentDraft[]; + pdfAttachments: PDFAttachmentDraft[]; + modelSelection: ModelSelection | null; + settingsOfProvider: SettingsOfProvider; + notificationService: NotificationServiceLike; +}; + +export type PrepareChatSubmitResult = + | { ok: true; images: ChatImageAttachment[]; pdfs: ChatPDFAttachment[] } + | { ok: false }; + +/** Convert composer attachments and validate vision support before send. */ +export const prepareChatSubmitAttachments = async ({ + imageAttachments, + pdfAttachments, + modelSelection, + settingsOfProvider, + notificationService, +}: PrepareChatSubmitParams): Promise => { + const images = toChatImageAttachments(imageAttachments); + const pdfs = toChatPDFAttachments(pdfAttachments); + + const processingNames = getProcessingPDFFilenames(pdfAttachments); + if (processingNames.length > 0) { + notificationService.warn( + `Some PDFs are still processing: ${processingNames.join(', ')}. They will be sent but may not have extracted text available yet.`, + ); + } + + if ((images.length === 0 && pdfs.length === 0) || !modelSelection) { + return { ok: true, images, pdfs }; + } + + const { + isSelectedModelVisionCapable, + checkOllamaModelVisionCapable, + hasVisionCapableApiKey, + hasOllamaVisionModel, + isOllamaAccessible, + } = await import('../../util/visionModelHelper.js'); + + if (modelSelection.providerName === 'auto' && modelSelection.modelName === 'auto') { + if (images.length > 0) { + const hasApiKey = hasVisionCapableApiKey(settingsOfProvider, modelSelection); + const ollamaAccessible = await isOllamaAccessible(); + const hasOllamaVision = ollamaAccessible && await hasOllamaVisionModel(); + if (!hasApiKey && !hasOllamaVision) { + notificationService.error( + 'No vision-capable models available. Please set up an API key (Anthropic, OpenAI, or Gemini) or install an Ollama vision model (e.g., llava, bakllava) to use images.', + ); + return { ok: false }; + } + } + return { ok: true, images, pdfs }; + } + + let isVisionCapable = isSelectedModelVisionCapable(modelSelection, settingsOfProvider); + if (!isVisionCapable && modelSelection.providerName === 'ollama') { + const ollamaAccessible = await isOllamaAccessible(); + if (ollamaAccessible) { + isVisionCapable = await checkOllamaModelVisionCapable(modelSelection.modelName); + } + } + + if (!isVisionCapable) { + const hasApiKey = hasVisionCapableApiKey(settingsOfProvider, modelSelection); + const ollamaAccessible = await isOllamaAccessible(); + const hasOllamaVision = ollamaAccessible && await hasOllamaVisionModel(); + if (!hasApiKey && !hasOllamaVision) { + notificationService.error( + 'The selected model does not support images or PDFs. Please select a vision-capable model (e.g., Claude, GPT-4, Gemini, or an Ollama vision model like llava).', + ); + return { ok: false }; + } + notificationService.warn( + 'The selected model may not support images or PDFs. Consider switching to a vision-capable model for better results.', + ); + } + + return { ok: true, images, pdfs }; +}; diff --git a/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/useVisionAttachmentHandlers.ts b/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/useVisionAttachmentHandlers.ts new file mode 100644 index 00000000000..a6bbbb7b4b7 --- /dev/null +++ b/src/vs/workbench/contrib/cortexide/browser/react/src/sidebar-tsx/composer/useVisionAttachmentHandlers.ts @@ -0,0 +1,92 @@ +/*-------------------------------------------------------------------------------------- + * Copyright 2025 Glass Devtools, Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. + *--------------------------------------------------------------------------------------*/ + +import { useCallback } from 'react'; +import { CortexideSettingsState } from '../../../../common/cortexideSettingsService.js'; + +type ServiceAccessor = { get: (id: string) => any }; + +type UseVisionAttachmentHandlersParams = { + settingsState: CortexideSettingsState; + accessor: ServiceAccessor; + settingsCommandId: string; + addImagesRaw: (files: File[]) => Promise; + addPDFsRaw: (files: File[]) => Promise; +}; + +/** Vision-aware wrappers for paste/drop image and PDF handlers in the chat composer. */ +export const useVisionAttachmentHandlers = ({ + settingsState, + accessor, + settingsCommandId, + addImagesRaw, + addPDFsRaw, +}: UseVisionAttachmentHandlersParams) => { + const addPDFs = useCallback(async (files: File[]) => { + const currentModelSel = settingsState.modelSelectionOfFeature['Chat']; + if (currentModelSel?.providerName === 'auto' && currentModelSel?.modelName === 'auto') { + await addPDFsRaw(files); + return; + } + await addPDFsRaw(files); + }, [addPDFsRaw, settingsState]); + + const addImages = useCallback(async (files: File[]) => { + const currentModelSel = settingsState.modelSelectionOfFeature['Chat']; + if (currentModelSel?.providerName === 'auto' && currentModelSel?.modelName === 'auto') { + await addImagesRaw(files); + return; + } + + const { + isSelectedModelVisionCapable, + checkOllamaModelVisionCapable, + hasVisionCapableApiKey, + hasOllamaVisionModel, + isOllamaAccessible, + } = await import('../../util/visionModelHelper.js'); + + let selectedIsVision = isSelectedModelVisionCapable(currentModelSel, settingsState.settingsOfProvider); + if (!selectedIsVision && currentModelSel?.providerName === 'ollama') { + const ollamaAccessible = await isOllamaAccessible(); + if (ollamaAccessible) { + selectedIsVision = await checkOllamaModelVisionCapable(currentModelSel.modelName); + } + } + + if (selectedIsVision) { + await addImagesRaw(files); + return; + } + + const hasApiKey = hasVisionCapableApiKey(settingsState.settingsOfProvider, currentModelSel); + const ollamaAccessible = await isOllamaAccessible(); + const hasOllamaVision = ollamaAccessible && await hasOllamaVisionModel(); + + if (!hasApiKey && !hasOllamaVision) { + const notificationService = accessor.get('INotificationService'); + const commandService = accessor.get('ICommandService'); + notificationService.notify({ + severity: 2, + message: 'No vision-capable models available. Please set up an API key (Anthropic, OpenAI, or Gemini) or install an Ollama vision model (e.g., llava, bakllava).', + actions: { + primary: [{ + id: 'void.vision.setup', + label: 'Setup Ollama Vision Models', + tooltip: '', + class: undefined, + enabled: true, + run: () => commandService.executeCommand(settingsCommandId), + }], + }, + }); + return; + } + + await addImagesRaw(files); + }, [addImagesRaw, settingsState, accessor, settingsCommandId]); + + return { addImages, addPDFs }; +}; diff --git a/src/vs/workbench/contrib/cortexide/common/prepareChatAttachments.ts b/src/vs/workbench/contrib/cortexide/common/prepareChatAttachments.ts new file mode 100644 index 00000000000..7ccb30cf2e8 --- /dev/null +++ b/src/vs/workbench/contrib/cortexide/common/prepareChatAttachments.ts @@ -0,0 +1,61 @@ +/*-------------------------------------------------------------------------------------- + * Copyright 2025 Glass Devtools, Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. + *--------------------------------------------------------------------------------------*/ + +import { ChatImageAttachment, ChatPDFAttachment } from './chatThreadServiceTypes.js'; + +export type ImageAttachmentDraft = { + id: string; + data: string; + mimeType: string; + filename: string; + width?: number; + height?: number; + size?: number; + uploadStatus?: string; +}; + +export type PDFAttachmentDraft = { + id: string; + data: string; + filename: string; + size?: number; + pageCount?: number; + selectedPages?: number[]; + extractedText?: string; + pagePreviews?: string[]; + uploadStatus?: string; +}; + +export const toChatImageAttachments = (attachments: ImageAttachmentDraft[]): ChatImageAttachment[] => + attachments + .filter(att => att.uploadStatus === 'success' || !att.uploadStatus) + .map(att => ({ + id: att.id, + data: att.data, + mimeType: att.mimeType, + filename: att.filename, + width: att.width, + height: att.height, + size: att.size, + })); + +export const toChatPDFAttachments = (attachments: PDFAttachmentDraft[]): ChatPDFAttachment[] => + attachments + .filter(att => att.uploadStatus !== 'failed') + .map(att => ({ + id: att.id, + data: att.data, + filename: att.filename, + size: att.size, + pageCount: att.pageCount, + selectedPages: att.selectedPages, + extractedText: att.extractedText, + pagePreviews: att.pagePreviews, + })); + +export const getProcessingPDFFilenames = (attachments: PDFAttachmentDraft[]): string[] => + attachments + .filter(att => att.uploadStatus === 'uploading' || att.uploadStatus === 'processing') + .map(p => p.filename); diff --git a/src/vs/workbench/contrib/cortexide/test/common/prepareChatAttachments.test.ts b/src/vs/workbench/contrib/cortexide/test/common/prepareChatAttachments.test.ts new file mode 100644 index 00000000000..6ca1ef9b977 --- /dev/null +++ b/src/vs/workbench/contrib/cortexide/test/common/prepareChatAttachments.test.ts @@ -0,0 +1,42 @@ +/*-------------------------------------------------------------------------------------- + * Copyright 2025 Glass Devtools, Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. + *--------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { suite, test } from 'mocha'; +import { + getProcessingPDFFilenames, + toChatImageAttachments, + toChatPDFAttachments, +} from '../../common/prepareChatAttachments.js'; + +suite('prepareChatAttachments', () => { + + test('toChatImageAttachments excludes failed uploads', () => { + const result = toChatImageAttachments([ + { id: '1', data: 'a', mimeType: 'image/png', filename: 'a.png', uploadStatus: 'failed' }, + { id: '2', data: 'b', mimeType: 'image/png', filename: 'b.png', uploadStatus: 'success' }, + { id: '3', data: 'c', mimeType: 'image/png', filename: 'c.png' }, + ]); + assert.strictEqual(result.length, 2); + assert.deepStrictEqual(result.map(r => r.id), ['2', '3']); + }); + + test('toChatPDFAttachments excludes failed uploads', () => { + const result = toChatPDFAttachments([ + { id: '1', data: 'a', filename: 'a.pdf', uploadStatus: 'failed' }, + { id: '2', data: 'b', filename: 'b.pdf', uploadStatus: 'processing' }, + ]); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].filename, 'b.pdf'); + }); + + test('getProcessingPDFFilenames lists in-flight PDFs', () => { + const names = getProcessingPDFFilenames([ + { id: '1', data: 'a', filename: 'done.pdf', uploadStatus: 'success' }, + { id: '2', data: 'b', filename: 'wait.pdf', uploadStatus: 'processing' }, + ]); + assert.deepStrictEqual(names, ['wait.pdf']); + }); +});