diff --git a/apps/desktop/src/main/__tests__/attachment-input-notice.test.ts b/apps/desktop/src/main/__tests__/attachment-input-notice.test.ts new file mode 100644 index 0000000000..d7cee0d9fa --- /dev/null +++ b/apps/desktop/src/main/__tests__/attachment-input-notice.test.ts @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { LlmConnection } from '@maka/core/llm-connections'; +import { shouldShowImageAttachmentNotDirectNotice } from '../../renderer/attachment-input-notice.js'; + +function connection(vision: boolean | undefined): LlmConnection { + return { + slug: 'test', + name: 'Test', + providerType: 'openai-compatible', + defaultModel: 'model', + enabled: true, + createdAt: 0, + updatedAt: 0, + models: [{ id: 'model', capabilities: vision === undefined ? undefined : { vision } }], + }; +} + +test('shows an advisory notice only for images on a non-vision model', () => { + const target = { connectionSlug: 'test', model: 'model' }; + assert.equal( + shouldShowImageAttachmentNotDirectNotice({ + attachments: [{ kind: 'image' }], + target, + connections: [connection(false)], + }), + true, + ); + assert.equal( + shouldShowImageAttachmentNotDirectNotice({ + attachments: [{ kind: 'image' }], + target, + connections: [connection(true)], + }), + false, + ); + assert.equal( + shouldShowImageAttachmentNotDirectNotice({ + attachments: [{ kind: 'other' }], + target, + connections: [connection(false)], + }), + false, + ); +}); diff --git a/apps/desktop/src/main/__tests__/new-task-staged-content.test.ts b/apps/desktop/src/main/__tests__/new-task-staged-content.test.ts index d201d9e996..a074403161 100644 --- a/apps/desktop/src/main/__tests__/new-task-staged-content.test.ts +++ b/apps/desktop/src/main/__tests__/new-task-staged-content.test.ts @@ -181,6 +181,28 @@ test('a completing send clears the attachments it submitted', async () => { assert.equal(probe.latest().pendingAttachments.length, 0); }); +test('reports newly staged image attachments without changing their staging behavior', async () => { + const stagedKinds: string[][] = []; + const probe = await mountProbe((options) => + useComposerAttachments({ + ...options, + toastApi: { error() {} }, + service: idleAttachmentService, + onAttachmentsStaged: (attachments) => stagedKinds.push(attachments.map((attachment) => attachment.kind)), + }), + ); + + await probe.render(NEW_TASK_PENDING_KEY); + await act(() => + probe.latest().attachFilePaths([ + { name: 'chart.png', type: 'image/png', size: 12 } as unknown as File, + ]), + ); + + assert.deepEqual(stagedKinds, [['image']]); + assert.equal(probe.latest().pendingAttachments[0]?.kind, 'image'); +}); + test('retracted queue attachments can be restored and submitted without re-ingest', async () => { const probe = await mountProbe((options) => useComposerAttachments({ diff --git a/apps/desktop/src/renderer/app-shell.tsx b/apps/desktop/src/renderer/app-shell.tsx index 458df497d9..6792d31ab6 100644 --- a/apps/desktop/src/renderer/app-shell.tsx +++ b/apps/desktop/src/renderer/app-shell.tsx @@ -212,6 +212,7 @@ import { useShellConnections } from './use-shell-connections'; import { useShellChatModel } from './use-shell-chat-model'; import { useShellLiveTurn } from './use-shell-live-turn'; import { useShellResume } from './use-shell-resume'; +import { shouldShowImageAttachmentNotDirectNotice } from './attachment-input-notice'; function rebaseWorkspaceFileReferences( sourceText: string, @@ -403,6 +404,16 @@ function AppShellContent({ draftKey: attachmentDraftKey, toastApi, service: window.maka.attachments, + onAttachmentsStaged(staged) { + const target = activeSession + ? { connectionSlug: activeSession.llmConnectionSlug, model: activeSession.model } + : newChatModel + ? { connectionSlug: newChatModel.llmConnectionSlug, model: newChatModel.model } + : undefined; + if (!shouldShowImageAttachmentNotDirectNotice({ attachments: staged, target, connections })) return; + const copy = getDesktopConversationCopy(uiLocale).actions; + toastApi.info(copy.imageAttachmentNotDirectTitle, copy.imageAttachmentNotDirectDescription); + }, }); const { pendingQuotes, diff --git a/apps/desktop/src/renderer/attachment-input-notice.ts b/apps/desktop/src/renderer/attachment-input-notice.ts new file mode 100644 index 0000000000..e64c8eea17 --- /dev/null +++ b/apps/desktop/src/renderer/attachment-input-notice.ts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { resolveModelVisionSupport } from '@maka/core/model-metadata'; +import { relayModelProfile } from '@maka/core/model-thinking'; +import type { LlmConnection } from '@maka/core/llm-connections'; + +type AttachmentCandidate = { kind: string }; + +/** + * Whether Desktop should explain that a newly staged image will not be + * delivered as a native provider image part. This is advisory only: the + * attachment remains staged and model-facing Read references remain intact. + */ +export function shouldShowImageAttachmentNotDirectNotice(input: { + attachments: readonly AttachmentCandidate[]; + target: { connectionSlug: string; model: string } | undefined; + connections: readonly LlmConnection[]; +}): boolean { + if (!input.attachments.some((attachment) => attachment.kind === 'image')) return false; + if (!input.target) return false; + const connection = input.connections.find((entry) => entry.slug === input.target!.connectionSlug); + if (!connection) return false; + return !resolveModelVisionSupport( + connection.providerType, + connection.models, + input.target.model, + relayModelProfile(connection, input.target.model)?.vision, + ); +} diff --git a/apps/desktop/src/renderer/locales/conversation-copy.ts b/apps/desktop/src/renderer/locales/conversation-copy.ts index 52496bd793..f116e3fa0a 100644 --- a/apps/desktop/src/renderer/locales/conversation-copy.ts +++ b/apps/desktop/src/renderer/locales/conversation-copy.ts @@ -52,6 +52,8 @@ export interface DesktopConversationCopy { operationFailedTitle: string; operationFailedFallback: string; attachmentFailedTitle: string; + imageAttachmentNotDirectTitle: string; + imageAttachmentNotDirectDescription: string; tryAgain: string; modelReboundTitle: string; modelReboundDescription: (modelId?: string) => string; @@ -416,7 +418,7 @@ function enDetail(parts: readonly string[]): string { const COPY = { zh: { - actions: { stopFailedTitle: '停止失败', stopFailedFallback: '任务操作失败，请稍后重试。', refreshSessionsFailedTitle: '刷新任务列表失败', refreshSessionsFailedFallback: '刷新任务列表失败，请稍后重试。', conversationErrorTitle: '任务出错', conversationErrorFallback: '任务运行失败，请稍后重试。', regenerateStartedTitle: '已发起重新生成', regenerateStartedDescription: '正在生成新的一轮回答', branchCreatedTitle: '已创建分支', branchCreatedDescription: (name) => `新任务 ${name}`, revisionStartedTitle: '已创建修改版草稿', revisionStartedDescription: '原任务仍会保留；修改后发送将在新版本中继续', revisionReadyTitle: '可以修改并重发了', revisionReadyDescription: '已回到该消息之前；编辑后发送即可', revisionUnavailableTitle: '暂时无法编辑这条消息', revisionAttachmentsUnsupported: '包含附件的历史消息暂不支持编辑并重发，请复制文字后新建消息。', revisionTransformedTextUnsupported: '通过显式技能发送的历史消息暂不支持编辑并重发，请复制文字后重新选择技能。', revisionDraftAttachmentConflict: 'Composer 中已有待发送附件，请先发送或移除附件，再编辑历史消息。', revisionCommandUnsupported: '修改消息时不能执行 /compact、/side 或编排命令，请取消修改后再试。', revisionAlreadyActive: '已有一条消息正在修改，请先发送或取消当前修改。', revisionCancelLabel: '取消', revisionBannerTitle: '正在修改已发送消息', revisionBannerDetail: '· 发送后创建新版本', revisionUnchanged: '内容没有变化。如需重新回答，请使用“重新生成”。', operationFailedTitle: '操作失败', operationFailedFallback: '任务操作失败，请稍后重试。', attachmentFailedTitle: '添加附件失败', tryAgain: '请稍后重试。', modelReboundTitle: '已切换到可用模型', modelReboundDescription: (modelId) => `原任务使用的连接已不可用${modelId ? ` · ${modelId}` : ''}`, messageReadFailedTitle: '读取任务失败', returnLatest: '返回最新消息', scrollMainToBottom: '滚动主对话到底部' }, + actions: { stopFailedTitle: '停止失败', stopFailedFallback: '任务操作失败，请稍后重试。', refreshSessionsFailedTitle: '刷新任务列表失败', refreshSessionsFailedFallback: '刷新任务列表失败，请稍后重试。', conversationErrorTitle: '任务出错', conversationErrorFallback: '任务运行失败，请稍后重试。', regenerateStartedTitle: '已发起重新生成', regenerateStartedDescription: '正在生成新的一轮回答', branchCreatedTitle: '已创建分支', branchCreatedDescription: (name) => `新任务 ${name}`, revisionStartedTitle: '已创建修改版草稿', revisionStartedDescription: '原任务仍会保留；修改后发送将在新版本中继续', revisionReadyTitle: '可以修改并重发了', revisionReadyDescription: '已回到该消息之前；编辑后发送即可', revisionUnavailableTitle: '暂时无法编辑这条消息', revisionAttachmentsUnsupported: '包含附件的历史消息暂不支持编辑并重发，请复制文字后新建消息。', revisionTransformedTextUnsupported: '通过显式技能发送的历史消息暂不支持编辑并重发，请复制文字后重新选择技能。', revisionDraftAttachmentConflict: 'Composer 中已有待发送附件，请先发送或移除附件，再编辑历史消息。', revisionCommandUnsupported: '修改消息时不能执行 /compact、/side 或编排命令，请取消修改后再试。', revisionAlreadyActive: '已有一条消息正在修改，请先发送或取消当前修改。', revisionCancelLabel: '取消', revisionBannerTitle: '正在修改已发送消息', revisionBannerDetail: '· 发送后创建新版本', revisionUnchanged: '内容没有变化。如需重新回答，请使用“重新生成”。', operationFailedTitle: '操作失败', operationFailedFallback: '任务操作失败，请稍后重试。', attachmentFailedTitle: '添加附件失败', imageAttachmentNotDirectTitle: '图片已作为附件添加', imageAttachmentNotDirectDescription: '当前模型不会直接接收图片。图片已作为附件提供给模型。', tryAgain: '请稍后重试。', modelReboundTitle: '已切换到可用模型', modelReboundDescription: (modelId) => `原任务使用的连接已不可用${modelId ? ` · ${modelId}` : ''}`, messageReadFailedTitle: '读取任务失败', returnLatest: '返回最新消息', scrollMainToBottom: '滚动主对话到底部' }, attachments: { tooMany: '附件数量超过 8 个', tooLarge: '附件大小超过 50MB', duplicate: '附件来源重复，请勿重复添加同一文件。' }, model: { fakeBackendLabel: '本地模拟连接', @@ -641,7 +643,7 @@ const COPY = { turnError: { unknown: '未知错误', contextOverflow: '上下文窗口已超出限制', contextBudgetExhausted: '上下文已达到上限，当前任务无法继续', malformedSummary: '上下文压缩未能生成有效摘要。请检查模型的上下文窗口设置、切换模型，或开启新任务。', timeout: '请求超时', auth: '鉴权失败', providerBilling: '模型服务计费受限', providerCapacity: '模型服务暂时满载，请稍后重试或切换模型', rateLimit: '触发模型速率限制', network: '网络错误', provider: '模型服务返回错误', stepCap: '达到工具步骤上限', tool: '工具调用失败', permission: '等待权限确认', restarted: '本地应用重启，上一轮没有完成', sandboxBoundaryClosed: '本地应用重启，等待确认的「允许访问工作区以外的内容」请求已按拒绝关闭', recovery: { safeResume: '检查当前状态后，可尝试安全恢复', stepCap: '任务可能尚未完成，可以继续', toolError: '先检查工具结果，再决定是否重试', connection: '先检查模型连接或登录状态', partial: '已保留部分输出，可从这里继续', toolRecord: '工具记录已保留，重试前先看结果', retry: '没有执行工具，可直接重试', capacity: '模型服务暂时满载，请等待几分钟或切换模型后重试', contextOverflow: '上下文仍超出限制，请减少附件或开启新任务', contextBudgetExhausted: '检查模型的上下文窗口设置、切换模型，或开启新任务', sandboxBoundaryClosed: '访问范围没有放开，重试本轮后可重新决定' } }, }, en: { - actions: { stopFailedTitle: 'Failed to stop', stopFailedFallback: 'The task action failed. Try again later.', refreshSessionsFailedTitle: 'Failed to refresh tasks', refreshSessionsFailedFallback: 'The task list could not be refreshed. Try again later.', conversationErrorTitle: 'Task error', conversationErrorFallback: 'The task run failed. Try again later.', regenerateStartedTitle: 'Regeneration started', regenerateStartedDescription: 'Generating a new response', branchCreatedTitle: 'Branch created', branchCreatedDescription: (name) => `New task: ${name}`, revisionStartedTitle: 'Edit draft ready', revisionStartedDescription: 'The original task is kept; sending creates a new version', revisionReadyTitle: 'Ready to edit and resend', revisionReadyDescription: 'Rewound to before that message; edit and send when ready', revisionUnavailableTitle: 'This message cannot be edited yet', revisionAttachmentsUnsupported: 'Edit & resend does not yet support historical attachments. Copy the text into a new message instead.', revisionTransformedTextUnsupported: 'Edit & resend does not yet support messages sent with an explicit skill. Copy the text and select the skill again instead.', revisionDraftAttachmentConflict: 'The composer already has pending attachments. Send or remove them before editing a sent message.', revisionCommandUnsupported: 'You cannot run /compact, /side, or orchestration commands while editing a sent message. Cancel the edit first.', revisionAlreadyActive: 'Another message is already being edited. Send or cancel that edit first.', revisionCancelLabel: 'Cancel', revisionBannerTitle: 'Editing sent message', revisionBannerDetail: '· New version on send', revisionUnchanged: 'Nothing changed. Use Regenerate if you only want a new answer.', operationFailedTitle: 'Action failed', operationFailedFallback: 'The task action failed. Try again later.', attachmentFailedTitle: 'Failed to add attachment', tryAgain: 'Try again later.', modelReboundTitle: 'Switched to an available model', modelReboundDescription: (modelId) => `The previous connection is unavailable${modelId ? ` · ${modelId}` : ''}`, messageReadFailedTitle: 'Failed to load task', returnLatest: 'Return to latest', scrollMainToBottom: 'Scroll main conversation to bottom' }, + actions: { stopFailedTitle: 'Failed to stop', stopFailedFallback: 'The task action failed. Try again later.', refreshSessionsFailedTitle: 'Failed to refresh tasks', refreshSessionsFailedFallback: 'The task list could not be refreshed. Try again later.', conversationErrorTitle: 'Task error', conversationErrorFallback: 'The task run failed. Try again later.', regenerateStartedTitle: 'Regeneration started', regenerateStartedDescription: 'Generating a new response', branchCreatedTitle: 'Branch created', branchCreatedDescription: (name) => `New task: ${name}`, revisionStartedTitle: 'Edit draft ready', revisionStartedDescription: 'The original task is kept; sending creates a new version', revisionReadyTitle: 'Ready to edit and resend', revisionReadyDescription: 'Rewound to before that message; edit and send when ready', revisionUnavailableTitle: 'This message cannot be edited yet', revisionAttachmentsUnsupported: 'Edit & resend does not yet support historical attachments. Copy the text into a new message instead.', revisionTransformedTextUnsupported: 'Edit & resend does not yet support messages sent with an explicit skill. Copy the text and select the skill again instead.', revisionDraftAttachmentConflict: 'The composer already has pending attachments. Send or remove them before editing a sent message.', revisionCommandUnsupported: 'You cannot run /compact, /side, or orchestration commands while editing a sent message. Cancel the edit first.', revisionAlreadyActive: 'Another message is already being edited. Send or cancel that edit first.', revisionCancelLabel: 'Cancel', revisionBannerTitle: 'Editing sent message', revisionBannerDetail: '· New version on send', revisionUnchanged: 'Nothing changed. Use Regenerate if you only want a new answer.', operationFailedTitle: 'Action failed', operationFailedFallback: 'The task action failed. Try again later.', attachmentFailedTitle: 'Failed to add attachment', imageAttachmentNotDirectTitle: 'Image added as an attachment', imageAttachmentNotDirectDescription: 'The current model does not receive images directly. The image has been provided as an attachment.', tryAgain: 'Try again later.', modelReboundTitle: 'Switched to an available model', modelReboundDescription: (modelId) => `The previous connection is unavailable${modelId ? ` · ${modelId}` : ''}`, messageReadFailedTitle: 'Failed to load task', returnLatest: 'Return to latest', scrollMainToBottom: 'Scroll main conversation to bottom' }, attachments: { tooMany: 'You can attach at most 8 files', tooLarge: 'Attachments must be 50 MB or smaller', duplicate: 'This attachment was already added.' }, model: { fakeBackendLabel: 'Local simulation', diff --git a/apps/desktop/src/renderer/use-composer-attachments.ts b/apps/desktop/src/renderer/use-composer-attachments.ts index b960ca2716..2e8e763625 100644 --- a/apps/desktop/src/renderer/use-composer-attachments.ts +++ b/apps/desktop/src/renderer/use-composer-attachments.ts @@ -121,6 +121,8 @@ export function useComposerAttachments(options: { draftKey: string; toastApi: ToastApi; service: ComposerAttachmentService; + /** Called after new items enter the visible composer draft. */ + onAttachmentsStaged?: (attachments: readonly PendingAttachment[]) => void; }) { const uiLocale = useUiLocale(); const copy = getDesktopConversationCopy(uiLocale).actions; @@ -136,9 +138,13 @@ export function useComposerAttachments(options: { // The live staging key, for the one import that resolves long after it was // started: the native file dialog. See pickAttachments. const draftKeyRef = useRef(options.draftKey); + // The model target can change while the native dialog is open. Keep this + // notification callback live for the same reason the draft owner is live. + const onAttachmentsStagedRef = useRef(options.onAttachmentsStaged); useEffect(() => { draftKeyRef.current = options.draftKey; - }, [options.draftKey]); + onAttachmentsStagedRef.current = options.onAttachmentsStaged; + }, [options.draftKey, options.onAttachmentsStaged]); const stagedAttachments = selectPending(pendingByKey, options.draftKey); const pendingAttachments = useMemo( () => @@ -219,6 +225,7 @@ export function useComposerAttachments(options: { const staged = result.files.map(approvalToPending); setPendingByKey((map) => appendPending(map, ownerKey, staged)); for (const item of staged) stagedKeysRef.current.add(item.stagingKey); + onAttachmentsStagedRef.current?.(staged); void loadPreviewsSequentially(staged); } catch (error) { options.toastApi.error( @@ -234,6 +241,7 @@ export function useComposerAttachments(options: { const staged = files.map(fileToPending); setPendingByKey((map) => appendPending(map, ownerKey, staged)); for (const item of staged) stagedKeysRef.current.add(item.stagingKey); + onAttachmentsStagedRef.current?.(staged); void loadPreviewsSequentially(staged); } diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 9c4b2a1d20..c1b9405244 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -2740,7 +2740,7 @@ describe('AiSdkBackend model history', () => { ); }); - test('current-turn image attachment falls back to text unless vision support is explicit', async () => { + test('current-turn image attachment keeps its Read reference unless vision support is explicit', async () => { const pngBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]); const model = completionModel(); const backend = createTestAiSdkBackend({ @@ -2767,7 +2767,7 @@ describe('AiSdkBackend model history', () => { name: 'chart.png', mimeType: 'image/png', bytes: pngBytes.length, - ref: { kind: 'session_file', sessionId: 'session-1', relativePath: 'fake/chart.png' }, + ref: { kind: 'session_file', sessionId: 'session-1', relativePath: 'artifact-1' }, }, ], context: [], @@ -2787,15 +2787,11 @@ describe('AiSdkBackend model history', () => { const text = parts.map((p) => p.text ?? '').join('\n'); assert.ok(text.includes('describe this chart'), `expected original text in: ${text}`); assert.ok( - text.includes( - '\nThe attachment content is unavailable to Read.\nname: "chart.png"\nmime_type: "image/png"\n', - ), - `expected unavailable attachment context in: ${text}`, - ); - assert.ok( - text.includes('does not support image input'), - `expected non-vision fallback note in: ${text}`, + text.includes('\nRead argument: {"ref":"maka://runtime/attachments/artifact-1"}'), + `expected attachment Read reference in: ${text}`, ); + assert.doesNotMatch(text, /does not support image input/); + assert.doesNotMatch(text, /switch to a vision-capable model/); }); test('reports unavailable attachment reads without consuming image budget', async () => { diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index dbb8c1c92f..51764874a5 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -860,10 +860,6 @@ export interface SystemPromptContext { emitSkillCatalogTrace?: (message: string, data?: Record) => void; } -function appendNonVisionImageFallbackNotice(textContent: string): string { - return `${textContent}\n\n[image attachments omitted: the selected model does not support image input. Tell the user you cannot view the attached image(s) and ask them to describe the image or switch to a vision-capable model.]`; -} - function isImageToolResult( value: unknown, ): value is { kind: 'image'; mimeType: string; ref: StorageRef } { @@ -4391,7 +4387,10 @@ export class AiSdkBackend implements AgentBackend { return textContent; } if (this.input.supportsVision !== true) { - return appendNonVisionImageFallbackNotice(textContent); + // `textContent` already carries each attachment's stable Read argument. + // Native provider image delivery is unavailable here, but that does not + // establish whether the model can process the image through a tool. + return textContent; } if (!this.input.readAttachmentBytes) { return textContent;