diff --git a/src/_locales/de/main.json b/src/_locales/de/main.json index 8b0d204bc..6d86fb0ab 100644 --- a/src/_locales/de/main.json +++ b/src/_locales/de/main.json @@ -82,6 +82,17 @@ "Model Name": "Modellname", "Custom Model API Url": "Benutzerdefinierte Modell-API-URL", "Loading...": "Laden...", + "Input tokens": "Eingabe-Tokens", + "Output tokens": "Ausgabe-Tokens", + "Total tokens": "Tokens insgesamt", + "Cached input tokens": "Zwischengespeicherte Eingabe-Tokens", + "Cache write tokens": "In den Cache geschriebene Tokens", + "Selected model": "Ausgewähltes Modell", + "Reported model": "Gemeldetes Modell", + "Model": "Modell", + "Models": "Modelle", + "Turns": "Runden", + "Reported usage": "Gemeldete Nutzung", "Feedback": "Feedback", "Confirm": "Bestätigen", "Clear Conversation": "Konversation löschen", diff --git a/src/_locales/en/main.json b/src/_locales/en/main.json index e558fc898..f8421ef92 100644 --- a/src/_locales/en/main.json +++ b/src/_locales/en/main.json @@ -89,6 +89,17 @@ "Model Name": "Model Name", "Custom Model API Url": "Custom Model API Url", "Loading...": "Loading...", + "Input tokens": "Input tokens", + "Output tokens": "Output tokens", + "Total tokens": "Total tokens", + "Cached input tokens": "Cached input tokens", + "Cache write tokens": "Cache write tokens", + "Selected model": "Selected model", + "Reported model": "Reported model", + "Model": "Model", + "Models": "Models", + "Turns": "Turns", + "Reported usage": "Reported usage", "Feedback": "Feedback", "Confirm": "Confirm", "Clear Conversation": "Clear Conversation", diff --git a/src/_locales/es/main.json b/src/_locales/es/main.json index c9f5b5a11..68ffadab7 100644 --- a/src/_locales/es/main.json +++ b/src/_locales/es/main.json @@ -80,6 +80,17 @@ "Model Name": "Nombre del modelo", "Custom Model API Url": "URL de la API de modelo personalizada", "Loading...": "Cargando...", + "Input tokens": "Tokens de entrada", + "Output tokens": "Tokens de salida", + "Total tokens": "Tokens totales", + "Cached input tokens": "Tokens de entrada en caché", + "Cache write tokens": "Tokens escritos en caché", + "Selected model": "Modelo seleccionado", + "Reported model": "Modelo informado", + "Model": "Modelo", + "Models": "Modelos", + "Turns": "Turnos", + "Reported usage": "Uso informado", "Feedback": "Comentarios", "Confirm": "Confirmar", "Clear Conversation": "Borrar conversación", diff --git a/src/_locales/fr/main.json b/src/_locales/fr/main.json index 1fa86e8a3..0b52d1a43 100644 --- a/src/_locales/fr/main.json +++ b/src/_locales/fr/main.json @@ -80,6 +80,17 @@ "Model Name": "Nom du modèle", "Custom Model API Url": "URL API personnalisée du modèle", "Loading...": "Chargement...", + "Input tokens": "Jetons d’entrée", + "Output tokens": "Jetons de sortie", + "Total tokens": "Total des jetons", + "Cached input tokens": "Jetons d’entrée mis en cache", + "Cache write tokens": "Jetons écrits dans le cache", + "Selected model": "Modèle sélectionné", + "Reported model": "Modèle déclaré", + "Model": "Modèle", + "Models": "Modèles", + "Turns": "Échanges", + "Reported usage": "Utilisation déclarée", "Feedback": "Commentaires", "Confirm": "Confirmer", "Clear Conversation": "Effacer la conversation", diff --git a/src/_locales/id/main.json b/src/_locales/id/main.json index 5267098d2..c4c571112 100644 --- a/src/_locales/id/main.json +++ b/src/_locales/id/main.json @@ -80,6 +80,17 @@ "Model Name": "Nama Model", "Custom Model API Url": "URL API Model Kustom", "Loading...": "Sedang Memuat...", + "Input tokens": "Token masukan", + "Output tokens": "Token keluaran", + "Total tokens": "Total token", + "Cached input tokens": "Token masukan dari cache", + "Cache write tokens": "Token yang ditulis ke cache", + "Selected model": "Model yang dipilih", + "Reported model": "Model yang dilaporkan", + "Model": "Model", + "Models": "Model", + "Turns": "Putaran", + "Reported usage": "Penggunaan yang dilaporkan", "Feedback": "Masukan", "Confirm": "Konfirmasi", "Clear Conversation": "Bersihkan Percakapan", diff --git a/src/_locales/it/main.json b/src/_locales/it/main.json index 4a592ff01..c6426dbc1 100644 --- a/src/_locales/it/main.json +++ b/src/_locales/it/main.json @@ -80,6 +80,17 @@ "Model Name": "Nome del modello", "Custom Model API Url": "URL API del modello personalizzato", "Loading...": "Caricamento...", + "Input tokens": "Token di input", + "Output tokens": "Token di output", + "Total tokens": "Token totali", + "Cached input tokens": "Token di input memorizzati nella cache", + "Cache write tokens": "Token scritti nella cache", + "Selected model": "Modello selezionato", + "Reported model": "Modello segnalato", + "Model": "Modello", + "Models": "Modelli", + "Turns": "Turni", + "Reported usage": "Utilizzo segnalato", "Feedback": "Feedback", "Confirm": "Conferma", "Clear Conversation": "Pulisci la conversazione", diff --git a/src/_locales/ja/main.json b/src/_locales/ja/main.json index 91c960949..8c9b67a2f 100644 --- a/src/_locales/ja/main.json +++ b/src/_locales/ja/main.json @@ -80,6 +80,17 @@ "Model Name": "モデル名", "Custom Model API Url": "カスタムモデルのAPI URL", "Loading...": "読み込み中...", + "Input tokens": "入力トークン", + "Output tokens": "出力トークン", + "Total tokens": "合計トークン", + "Cached input tokens": "キャッシュ済み入力トークン", + "Cache write tokens": "キャッシュ書き込みトークン", + "Selected model": "選択したモデル", + "Reported model": "報告されたモデル", + "Model": "モデル", + "Models": "モデル", + "Turns": "ターン数", + "Reported usage": "報告された使用量", "Feedback": "フィードバック", "Confirm": "確認", "Clear Conversation": "会話をクリア", diff --git a/src/_locales/ko/main.json b/src/_locales/ko/main.json index 252ed15f8..3f9b04a16 100644 --- a/src/_locales/ko/main.json +++ b/src/_locales/ko/main.json @@ -80,6 +80,17 @@ "Model Name": "모델 이름", "Custom Model API Url": "사용자 정의 모델 API URL", "Loading...": "로딩 중...", + "Input tokens": "입력 토큰", + "Output tokens": "출력 토큰", + "Total tokens": "총 토큰", + "Cached input tokens": "캐시된 입력 토큰", + "Cache write tokens": "캐시 쓰기 토큰", + "Selected model": "선택한 모델", + "Reported model": "보고된 모델", + "Model": "모델", + "Models": "모델", + "Turns": "턴 수", + "Reported usage": "보고된 사용량", "Feedback": "피드백", "Confirm": "확인", "Clear Conversation": "대화 내용 지우기", diff --git a/src/_locales/pt/main.json b/src/_locales/pt/main.json index b3379c159..803104582 100644 --- a/src/_locales/pt/main.json +++ b/src/_locales/pt/main.json @@ -80,6 +80,17 @@ "Model Name": "Nome do Modelo", "Custom Model API Url": "URL da API do Modelo Personalizado", "Loading...": "Carregando...", + "Input tokens": "Tokens de entrada", + "Output tokens": "Tokens de saída", + "Total tokens": "Total de tokens", + "Cached input tokens": "Tokens de entrada em cache", + "Cache write tokens": "Tokens gravados no cache", + "Selected model": "Modelo selecionado", + "Reported model": "Modelo informado", + "Model": "Modelo", + "Models": "Modelos", + "Turns": "Turnos", + "Reported usage": "Uso informado", "Feedback": "Feedback", "Confirm": "Confirmar", "Clear Conversation": "Limpar Conversa", diff --git a/src/_locales/ru/main.json b/src/_locales/ru/main.json index d857f3f62..8d6c56892 100644 --- a/src/_locales/ru/main.json +++ b/src/_locales/ru/main.json @@ -88,6 +88,17 @@ "Model Name": "Название модели", "Custom Model API Url": "Custom Model API Url", "Loading...": "Загрузка...", + "Input tokens": "Входные токены", + "Output tokens": "Выходные токены", + "Total tokens": "Всего токенов", + "Cached input tokens": "Кэшированные входные токены", + "Cache write tokens": "Токены, записанные в кэш", + "Selected model": "Выбранная модель", + "Reported model": "Модель, указанная провайдером", + "Model": "Модель", + "Models": "Модели", + "Turns": "Ходы", + "Reported usage": "Данные об использовании", "Feedback": "Обратная связь", "Confirm": "Подтверждение", "Clear Conversation": "Очистить беседу", diff --git a/src/_locales/tr/main.json b/src/_locales/tr/main.json index 1b2391cf4..01edc3048 100644 --- a/src/_locales/tr/main.json +++ b/src/_locales/tr/main.json @@ -80,6 +80,17 @@ "Model Name": "Model Adı", "Custom Model API Url": "Özel Model API Url'si", "Loading...": "Yükleniyor...", + "Input tokens": "Girdi tokenleri", + "Output tokens": "Çıktı tokenleri", + "Total tokens": "Toplam token", + "Cached input tokens": "Önbelleğe alınmış girdi tokenleri", + "Cache write tokens": "Önbelleğe yazılan tokenler", + "Selected model": "Seçilen model", + "Reported model": "Bildirilen model", + "Model": "Model", + "Models": "Modeller", + "Turns": "Turlar", + "Reported usage": "Bildirilen kullanım", "Feedback": "Geri Bildirim", "Confirm": "Onayla", "Clear Conversation": "Konuşmayı Temizle", diff --git a/src/_locales/zh-hans/main.json b/src/_locales/zh-hans/main.json index cfad6d2d1..4025b3fd4 100644 --- a/src/_locales/zh-hans/main.json +++ b/src/_locales/zh-hans/main.json @@ -83,6 +83,17 @@ "Model Name": "模型名", "Custom Model API Url": "自定义模型的API地址", "Loading...": "正在读取...", + "Input tokens": "输入 token", + "Output tokens": "输出 token", + "Total tokens": "token 总计", + "Cached input tokens": "缓存命中 token", + "Cache write tokens": "写入缓存 token", + "Selected model": "选用模型", + "Reported model": "上游回报模型", + "Model": "模型", + "Models": "模型", + "Turns": "轮次", + "Reported usage": "已回报用量", "Feedback": "反馈", "Confirm": "确认", "Clear Conversation": "清理对话", diff --git a/src/_locales/zh-hant/main.json b/src/_locales/zh-hant/main.json index 3dd20a229..572371ab3 100644 --- a/src/_locales/zh-hant/main.json +++ b/src/_locales/zh-hant/main.json @@ -83,6 +83,17 @@ "Model Name": "模型名稱", "Custom Model API Url": "自訂模型 API 網址", "Loading...": "載入中...", + "Input tokens": "輸入 token", + "Output tokens": "輸出 token", + "Total tokens": "token 合計", + "Cached input tokens": "快取命中 token", + "Cache write tokens": "寫入快取 token", + "Selected model": "選用模型", + "Reported model": "上游回報模型", + "Model": "模型", + "Models": "模型", + "Turns": "輪次", + "Reported usage": "已回報用量", "Feedback": "意見回饋", "Confirm": "確認", "Clear Conversation": "清除對話", diff --git a/src/components/ConversationCard/index.jsx b/src/components/ConversationCard/index.jsx index 6835faa6a..8bd80465e 100644 --- a/src/components/ConversationCard/index.jsx +++ b/src/components/ConversationCard/index.jsx @@ -3,6 +3,7 @@ import PropTypes from 'prop-types' import Browser from 'webextension-polyfill' import InputBox from '../InputBox' import ConversationItem from '../ConversationItem' +import ConversationUsageSummary from '../ConversationUsageSummary' import { apiModeToModelName, createElementAtPosition, @@ -45,6 +46,7 @@ import { createConversationPortMessage, createRetrySession, finalizeInterruptedSession, + getCompletedAnswerMetadata, getCompletedAnswerUpdate, getInterruptedCompletionState, isSupersededGenerationMessage, @@ -59,12 +61,14 @@ class ConversationItemData extends Object { * @param {'question'|'answer'|'error'} type * @param {string} content * @param {bool} done + * @param {Object|null} meta */ - constructor(type, content, done = false) { + constructor(type, content, done = false, meta = null) { super() this.type = type this.content = content this.done = done + this.meta = meta } } @@ -81,6 +85,7 @@ function ConversationCard(props) { const retryRecordRef = useRef(null) const retryGenerationIdRef = useRef(0) const requestGenerationIdRef = useRef(0) + const requestModelRef = useRef('') const [completeDraggable, setCompleteDraggable] = useState(false) const useForegroundFetch = isUsingBingWebModel(session) const [apiModes, setApiModes] = useState([]) @@ -121,7 +126,7 @@ function ConversationCard(props) { const ret = [] for (const record of session.conversationRecords) { ret.push(new ConversationItemData('question', record.question, true)) - ret.push(new ConversationItemData('answer', record.answer, true)) + ret.push(new ConversationItemData('answer', record.answer, true, record.meta)) } setConversationItemData(ret) } @@ -173,8 +178,9 @@ function ConversationCard(props) { * @param {boolean} appended * @param {'question'|'answer'|'error'} newType * @param {boolean} done + * @param {Object|null|undefined} meta */ - const updateAnswer = (value, appended, newType, done = false) => { + const updateAnswer = (value, appended, newType, done = false, meta = undefined) => { setConversationItemData((old) => { const copy = [...old] const index = findLastIndex(copy, (v) => v.type === 'answer' || v.type === 'error') @@ -182,8 +188,9 @@ function ConversationCard(props) { copy[index] = new ConversationItemData( newType, appended ? copy[index].content + value : value, + done, + meta === undefined ? copy[index].meta : meta, ) - copy[index].done = done return copy }) } @@ -211,7 +218,16 @@ function ConversationCard(props) { partialAnswerRef.current = '' retryRecordRef.current = null const answerUpdate = getCompletedAnswerUpdate(completionState.restoredRetryAnswer) - updateAnswer(answerUpdate.value, answerUpdate.appended, 'answer', true) + const answerMetadata = getCompletedAnswerMetadata({ + message: msg, + restoredRetryAnswer: completionState.restoredRetryAnswer, + partialAnswer, + retryRecord, + requestedModel: requestModelRef.current, + fallbackModel: currentAiName, + }) + requestModelRef.current = '' + updateAnswer(answerUpdate.value, answerUpdate.appended, 'answer', true, answerMetadata) setIsReady(true) } if (msg.error) { @@ -274,6 +290,7 @@ function ConversationCard(props) { } partialAnswerRef.current = '' retryRecordRef.current = null + requestModelRef.current = '' setIsReady(true) } } @@ -287,6 +304,9 @@ function ConversationCard(props) { */ const postMessage = async ({ session, stop, stopGenerationId }) => { const requestGenerationId = session ? ++requestGenerationIdRef.current : undefined + if (session) { + requestModelRef.current = getConversationAiName(session, t, customOpenAIProviders) + } if (useForegroundFetch) { foregroundMessageListeners.current.forEach((listener) => listener({ session, stop, stopGenerationId, requestGenerationId }), @@ -386,7 +406,13 @@ function ConversationCard(props) { }, [port, conversationItemData]) const getRetryFn = (session) => async () => { - updateAnswer(`

${t('Waiting for response...')}

`, false, 'answer') + updateAnswer( + `

${t('Waiting for response...')}

`, + false, + 'answer', + false, + null, + ) setIsReady(false) const conversationRecords = session.conversationRecords.map((record) => ({ ...record })) @@ -574,6 +600,7 @@ function ConversationCard(props) { } partialAnswerRef.current = '' retryRecordRef.current = null + requestModelRef.current = '' Browser.runtime.sendMessage({ type: 'DELETE_CONVERSATION', data: { @@ -649,6 +676,7 @@ function ConversationCard(props) {
+
))} diff --git a/src/components/ConversationCard/session.mjs b/src/components/ConversationCard/session.mjs index 40c60f166..991ff5fc6 100644 --- a/src/components/ConversationCard/session.mjs +++ b/src/components/ConversationCard/session.mjs @@ -1,9 +1,34 @@ import { pushRecord } from '../../services/apis/shared.mjs' +export function getLastConversationRecord(records) { + if (!Array.isArray(records) || records.length === 0) return null + return records[records.length - 1] +} + +export function getCompletedAnswerMetadata({ + message, + restoredRetryAnswer, + partialAnswer, + retryRecord, + requestedModel, + fallbackModel, +}) { + if (restoredRetryAnswer !== null) return retryRecord?.meta || null + + const responseMetadata = getLastConversationRecord( + message.session?.conversationRecords, + )?.meta + if (responseMetadata) return responseMetadata + + const selectedModel = requestedModel || fallbackModel + if (partialAnswer && selectedModel) return { selectedModel } + return undefined +} + export function finalizeInterruptedSession(session, answer, retryRecord = null) { if (!answer) { if (!session.isRetry && !retryRecord) return session - const lastRecord = session.conversationRecords.at(-1) + const lastRecord = getLastConversationRecord(session.conversationRecords) const shouldRestoreRetryRecord = retryRecord && (lastRecord?.question !== retryRecord.question || lastRecord?.answer !== retryRecord.answer) diff --git a/src/components/ConversationItem/index.jsx b/src/components/ConversationItem/index.jsx index d9e9e9f35..e44c57cd4 100644 --- a/src/components/ConversationItem/index.jsx +++ b/src/components/ConversationItem/index.jsx @@ -5,18 +5,72 @@ import ReadButton from '../ReadButton' import PropTypes from 'prop-types' import MarkdownRender from '../MarkdownRender/markdown.jsx' import { useTranslation } from 'react-i18next' +import { modelNameToDesc } from '../../utils/model-name-convert.mjs' +import { formatTokenCount, getRecordModel } from '../../utils/usage-metadata.mjs' -function AnswerTitle({ descName }) { +function getUsageText(meta, t) { + const usage = meta?.usage + if (!usage) return '' + + const parts = [] + if (usage.inputTokens !== undefined) + parts.push(`${t('Input tokens')}: ${formatTokenCount(usage.inputTokens)}`) + if (usage.outputTokens !== undefined) + parts.push(`${t('Output tokens')}: ${formatTokenCount(usage.outputTokens)}`) + if (usage.cacheReadInputTokens !== undefined) + parts.push(`${t('Cached input tokens')}: ${formatTokenCount(usage.cacheReadInputTokens)}`) + if (usage.cacheWriteInputTokens !== undefined) + parts.push(`${t('Cache write tokens')}: ${formatTokenCount(usage.cacheWriteInputTokens)}`) + if (parts.length === 0 && usage.totalTokens !== undefined) { + parts.push(`${t('Total tokens')}: ${formatTokenCount(usage.totalTokens)}`) + } + return parts.join(' · ') +} + +function AnswerTitle({ descName, meta }) { const { t } = useTranslation() + const model = modelNameToDesc(getRecordModel(meta, descName), t) + const usageText = getUsageText(meta, t) + const selectedModel = meta?.selectedModel + ? modelNameToDesc(meta.selectedModel, t) + : '' + const reportedModel = meta?.reportedModel + ? modelNameToDesc(meta.reportedModel, t) + : '' + const hasDifferentReportedModel = + meta?.selectedModel && + meta?.reportedModel && + meta.selectedModel !== meta.reportedModel + const modelTitle = hasDifferentReportedModel + ? `${t('Selected model')}: ${selectedModel}\n${t('Reported model')}: ${reportedModel}` + : model - return

{descName ? `${descName}:` : t('Loading...')}

+ return ( +
+

+ {model ? `${model}:` : t('Loading...')} +

+ {usageText && ( +

+ {usageText} +

+ )} +
+ ) } AnswerTitle.propTypes = { descName: PropTypes.string, + meta: PropTypes.object, } -export function ConversationItem({ type, content, descName, onRetry }) { +export function ConversationItem({ type, content, descName, meta, onRetry }) { const { t } = useTranslation() const [collapsed, setCollapsed] = useState(false) @@ -55,7 +109,7 @@ export function ConversationItem({ type, content, descName, onRetry }) { return (
- +
{onRetry && ( @@ -129,6 +183,7 @@ ConversationItem.propTypes = { type: PropTypes.oneOf(['question', 'answer', 'error']).isRequired, content: PropTypes.string.isRequired, descName: PropTypes.string, + meta: PropTypes.object, onRetry: PropTypes.func, } diff --git a/src/components/ConversationUsageSummary/index.jsx b/src/components/ConversationUsageSummary/index.jsx new file mode 100644 index 000000000..6785221a1 --- /dev/null +++ b/src/components/ConversationUsageSummary/index.jsx @@ -0,0 +1,126 @@ +import PropTypes from 'prop-types' +import { useTranslation } from 'react-i18next' +import { modelNameToDesc } from '../../utils/model-name-convert.mjs' +import { + formatTokenCount, + summarizeConversationUsage, +} from '../../utils/usage-metadata.mjs' + +function getMetricText(label, value, reportedTurns, totalReportedTurns, t) { + if (reportedTurns === 0) return null + const coverage = + reportedTurns === totalReportedTurns + ? '' + : ` (${t('Turns')}: ${reportedTurns}/${totalReportedTurns})` + return `${label}: ${formatTokenCount(value)}${coverage}` +} + +function ConversationUsageSummary({ records }) { + const { t } = useTranslation() + const summary = summarizeConversationUsage(records) + if (summary.models.length === 0 && summary.reportedTurns === 0) return null + + const models = summary.models.map(({ name, turns }) => ({ + name: modelNameToDesc(name, t), + turns, + })) + const hasModels = models.length > 0 + const modelReportedTurns = models.reduce((total, { turns }) => total + turns, 0) + const modelCoverage = + modelReportedTurns === summary.totalTurns + ? '' + : ` (${t('Turns')}: ${modelReportedTurns}/${summary.totalTurns})` + const modelText = !hasModels + ? '' + : models.length === 1 + ? `${t('Model')}: ${models[0].name}${modelCoverage}` + : `${t('Models')}: ${models.length}${modelCoverage}` + const modelTitle = models + .map(({ name, turns }) => `${name} · ${t('Turns')}: ${turns}`) + .join('\n') + const usageParts = [ + { + key: 'reported-usage', + text: + summary.reportedTurns > 0 + ? `${t('Reported usage')}: ${summary.reportedTurns}/${summary.totalTurns}` + : null, + }, + { + key: 'input-tokens', + text: getMetricText( + t('Input tokens'), + summary.inputTokens, + summary.inputReportedTurns, + summary.reportedTurns, + t, + ), + }, + { + key: 'output-tokens', + text: getMetricText( + t('Output tokens'), + summary.outputTokens, + summary.outputReportedTurns, + summary.reportedTurns, + t, + ), + }, + { + key: 'total-tokens', + text: getMetricText( + t('Total tokens'), + summary.totalTokens, + summary.totalReportedTurns, + summary.reportedTurns, + t, + ), + }, + { + key: 'cache-read-tokens', + text: getMetricText( + t('Cached input tokens'), + summary.cacheReadInputTokens, + summary.cacheReadReportedTurns, + summary.reportedTurns, + t, + ), + }, + { + key: 'cache-write-tokens', + text: getMetricText( + t('Cache write tokens'), + summary.cacheWriteInputTokens, + summary.cacheWriteReportedTurns, + summary.reportedTurns, + t, + ), + }, + ] + + return ( +
+ {hasModels && {modelText}} + {usageParts.map(({ key, text }) => { + if (!text) return null + return {text} + })} +
+ ) +} + +ConversationUsageSummary.propTypes = { + records: PropTypes.array, +} + +export default ConversationUsageSummary diff --git a/src/services/apis/claude-api.mjs b/src/services/apis/claude-api.mjs index d589fc894..f018782c6 100644 --- a/src/services/apis/claude-api.mjs +++ b/src/services/apis/claude-api.mjs @@ -5,6 +5,7 @@ import { isEmpty } from 'lodash-es' import { getConversationPairs } from '../../utils/get-conversation-pairs.mjs' import { getModelValue } from '../../utils/model-name-convert.mjs' import { getTemperatureParams } from './temperature-params.mjs' +import { mergeClaudeResponseMetadata } from '../../utils/usage-metadata.mjs' function shouldDisableDefaultThinking(model) { return model === 'claude-sonnet-5' @@ -16,7 +17,13 @@ function shouldDisableDefaultThinking(model) { * @param {Session} session */ export async function generateAnswersWithClaudeApi(port, question, session) { - const { controller, messageListener, disconnectListener } = setAbortController(port) + const { + controller, + messageListener, + disconnectListener, + getStopGenerationId, + isCurrentSessionRequest, + } = setAbortController(port) const config = await getUserConfig() const apiUrl = config.customAnthropicApiUrl const model = getModelValue(session) @@ -39,6 +46,7 @@ export async function generateAnswersWithClaudeApi(port, question, session) { } let answer = '' + let responseMetadata = null let stopReason = '' let completionError let wasAborted = false @@ -70,6 +78,7 @@ export async function generateAnswersWithClaudeApi(port, question, session) { throw error } if (completedSuccessfully) return + responseMetadata = mergeClaudeResponseMetadata(responseMetadata, data, model) if (data?.type === 'message_delta') { stopReason = data?.delta?.stop_reason || stopReason return @@ -95,7 +104,7 @@ export async function generateAnswersWithClaudeApi(port, question, session) { controller.abort() throw completionError } - pushRecord(session, question, answer) + pushRecord(session, question, answer, responseMetadata) console.debug('conversation history', { content: session.conversationRecords }) port.postMessage({ answer: null, done: true, session: session }) completedSuccessfully = true @@ -125,7 +134,24 @@ export async function generateAnswersWithClaudeApi(port, question, session) { throw new Error(!isEmpty(error) ? JSON.stringify(error) : `${resp.status} ${resp.statusText}`) }, }).catch((error) => error) - if (wasAborted) return + if (wasAborted) { + const shouldPostSession = Boolean(answer) || session.isRetry + if (shouldPostSession && isCurrentSessionRequest()) { + if (answer) pushRecord(session, question, answer, responseMetadata) + session.isRetry = false + try { + const stoppedGenerationId = getStopGenerationId() + port.postMessage({ + session, + ...(answer && responseMetadata ? { done: true } : {}), + ...(stoppedGenerationId === undefined ? {} : { stoppedGenerationId }), + }) + } catch (error) { + console.warn('[claude-api] Failed to post session on abort:', error) + } + } + return + } if (completionError) throw completionError if ( streamError && diff --git a/src/services/apis/openai-compatible-core.mjs b/src/services/apis/openai-compatible-core.mjs index 02dfe4a81..df5c9c96a 100644 --- a/src/services/apis/openai-compatible-core.mjs +++ b/src/services/apis/openai-compatible-core.mjs @@ -4,6 +4,7 @@ import { isEmpty } from 'lodash-es' import { getCompletionPromptBase, pushRecord, setAbortController } from './shared.mjs' import { getChatCompletionsTokenParams } from './openai-token-params.mjs' import { getTemperatureParams } from './temperature-params.mjs' +import { mergeOpenAIResponseMetadata } from '../../utils/usage-metadata.mjs' function buildHeaders(apiKey, extraHeaders = {}) { const headers = { @@ -28,10 +29,41 @@ function buildMessageAnswer(answer, data, allowLegacyResponseField) { return answer } +function hasMessageAnswerField(data, allowLegacyResponseField) { + if (allowLegacyResponseField && typeof data?.response === 'string') return true + + const choice = data?.choices?.[0] + return ( + typeof choice?.delta?.content === 'string' || + typeof choice?.message?.content === 'string' || + typeof choice?.text === 'string' + ) +} + function hasFinished(data) { return Boolean(data?.choices?.[0]?.finish_reason) } +function getRequestUrl(requestUrl) { + try { + return new URL(requestUrl) + } catch { + return null + } +} + +function isNativeOpenAIChatCompletionsRequest(requestUrl, endpointType) { + if (endpointType !== 'chat') return false + const url = getRequestUrl(requestUrl) + if (!url || url.hostname.toLowerCase() !== 'api.openai.com') return false + return url.pathname.replace(/\/+$/, '') === '/v1/chat/completions' +} + +function isOpenRouterRequest(requestUrl, provider) { + if (provider === 'openrouter') return true + return getRequestUrl(requestUrl)?.origin === 'https://openrouter.ai' +} + /** * @param {object} params * @param {Browser.Runtime.Port} params.port @@ -112,14 +144,27 @@ export async function generateAnswersWithOpenAICompatible({ ...getTemperatureParams(config, model), ...safeExtraBody, } + if (isNativeOpenAIChatCompletionsRequest(requestUrl, endpointType)) { + requestBody.stream_options = { + ...(requestBody.stream_options && typeof requestBody.stream_options === 'object' + ? requestBody.stream_options + : {}), + include_usage: true, + } + } } let answer = '' + let responseMetadata = null + let sawFinishReason = false let finished = false + const waitForFinalUsage = + isNativeOpenAIChatCompletionsRequest(requestUrl, endpointType) || + isOpenRouterRequest(requestUrl, provider) const finish = () => { if (finished) return finished = true - pushRecord(session, question, answer) + pushRecord(session, question, answer, responseMetadata) port.postMessage({ answer: null, done: true, session: session }) } @@ -142,10 +187,20 @@ export async function generateAnswersWithOpenAICompatible({ return } + responseMetadata = mergeOpenAIResponseMetadata(responseMetadata, data, model) + const previousAnswer = answer + const hasAnswerField = hasMessageAnswerField(data, allowLegacyResponseField) answer = buildMessageAnswer(answer, data, allowLegacyResponseField) - port.postMessage({ answer: answer, done: false, session: null }) + if (answer !== previousAnswer || hasAnswerField) { + port.postMessage({ answer: answer, done: false, session: null }) + } - if (hasFinished(data)) { + const chunkFinished = hasFinished(data) + const hasUsage = Boolean(data?.usage && typeof data.usage === 'object') + if (chunkFinished) sawFinishReason = true + if (chunkFinished && (!waitForFinalUsage || hasUsage)) { + finish() + } else if (sawFinishReason && waitForFinalUsage && hasUsage) { finish() } }, @@ -157,13 +212,14 @@ export async function generateAnswersWithOpenAICompatible({ const shouldPostSession = Boolean(answer) || session.isRetry if (shouldPostSession && isCurrentSessionRequest()) { if (answer) { - pushRecord(session, question, answer) + pushRecord(session, question, answer, responseMetadata) } session.isRetry = false try { const stoppedGenerationId = getStopGenerationId() port.postMessage({ session, + ...(answer && responseMetadata ? { done: true } : {}), ...(stoppedGenerationId === undefined ? {} : { stoppedGenerationId }), }) } catch (e) { @@ -182,6 +238,10 @@ export async function generateAnswersWithOpenAICompatible({ async onError(resp) { port.onMessage.removeListener(messageListener) port.onDisconnect.removeListener(disconnectListener) + if (sawFinishReason && waitForFinalUsage) { + finish() + return + } if (resp instanceof Error) throw resp const error = await resp.json().catch(() => ({})) throw new Error(!isEmpty(error) ? JSON.stringify(error) : `${resp.status} ${resp.statusText}`) diff --git a/src/services/apis/shared.mjs b/src/services/apis/shared.mjs index a425305f1..65cf373aa 100644 --- a/src/services/apis/shared.mjs +++ b/src/services/apis/shared.mjs @@ -1,3 +1,5 @@ +import { createRecordMetadata } from '../../utils/usage-metadata.mjs' + export const getChatSystemPromptBase = async () => { return `You are a helpful, creative, clever, and very friendly assistant. You are familiar with various languages in the world.` } @@ -75,11 +77,19 @@ export function setAbortController(port, onStop, onDisconnect) { } } -export function pushRecord(session, question, answer) { +export function pushRecord(session, question, answer, metadata = null) { + const recordMetadata = createRecordMetadata(session, metadata) const recordLength = session.conversationRecords.length let lastRecord if (recordLength > 0) lastRecord = session.conversationRecords[recordLength - 1] - if (session.isRetry && lastRecord && lastRecord.question === question) lastRecord.answer = answer - else session.conversationRecords.push({ question: question, answer: answer }) + if (session.isRetry && lastRecord && lastRecord.question === question) { + lastRecord.answer = answer + if (recordMetadata) lastRecord.meta = recordMetadata + else delete lastRecord.meta + } else { + const record = { question: question, answer: answer } + if (recordMetadata) record.meta = recordMetadata + session.conversationRecords.push(record) + } } diff --git a/src/utils/usage-metadata.mjs b/src/utils/usage-metadata.mjs new file mode 100644 index 000000000..f3b0bd8a7 --- /dev/null +++ b/src/utils/usage-metadata.mjs @@ -0,0 +1,214 @@ +const TOKEN_USAGE_KEYS = [ + 'inputTokens', + 'outputTokens', + 'totalTokens', + 'cacheReadInputTokens', + 'cacheWriteInputTokens', +] + +function toTokenCount(value) { + if (typeof value === 'string' && value.trim()) value = Number(value) + if (!Number.isFinite(value) || value < 0) return undefined + return Math.trunc(value) +} + +function firstTokenCount(...values) { + for (const value of values) { + const tokenCount = toTokenCount(value) + if (tokenCount !== undefined) return tokenCount + } + return undefined +} + +function toNonEmptyString(value) { + return typeof value === 'string' && value.trim() ? value.trim() : undefined +} + +function compactUsage(usage) { + if (!usage || typeof usage !== 'object') return null + const compacted = {} + for (const key of TOKEN_USAGE_KEYS) { + const value = toTokenCount(usage[key]) + if (value !== undefined) compacted[key] = value + } + if ( + compacted.totalTokens === undefined && + compacted.inputTokens !== undefined && + compacted.outputTokens !== undefined + ) { + compacted.totalTokens = compacted.inputTokens + compacted.outputTokens + } + return Object.keys(compacted).length > 0 ? compacted : null +} + +export function mergeResponseMetadata(currentMetadata, nextMetadata) { + const current = currentMetadata && typeof currentMetadata === 'object' ? currentMetadata : {} + const next = nextMetadata && typeof nextMetadata === 'object' ? nextMetadata : {} + const merged = {} + + const selectedModel = + toNonEmptyString(next.selectedModel) || toNonEmptyString(current.selectedModel) + const reportedModel = + toNonEmptyString(next.reportedModel) || toNonEmptyString(current.reportedModel) + if (selectedModel) merged.selectedModel = selectedModel + if (reportedModel) merged.reportedModel = reportedModel + + const currentUsage = compactUsage(current.usage) + const nextUsage = compactUsage(next.usage) + const mergedUsage = { ...(currentUsage || {}), ...(nextUsage || {}) } + if ( + nextUsage && + nextUsage.totalTokens === undefined && + (nextUsage.inputTokens !== undefined || nextUsage.outputTokens !== undefined) + ) { + delete mergedUsage.totalTokens + } + const usage = compactUsage(mergedUsage) + if (usage) merged.usage = usage + + return Object.keys(merged).length > 0 ? merged : null +} + +export function mergeOpenAIResponseMetadata(currentMetadata, data, selectedModel) { + const usage = data?.usage + const promptDetails = usage?.prompt_tokens_details + const inputDetails = usage?.input_tokens_details + const normalizedUsage = usage + ? compactUsage({ + inputTokens: firstTokenCount(usage.prompt_tokens, usage.input_tokens), + outputTokens: firstTokenCount(usage.completion_tokens, usage.output_tokens), + totalTokens: firstTokenCount(usage.total_tokens), + cacheReadInputTokens: firstTokenCount( + promptDetails?.cached_tokens, + inputDetails?.cached_tokens, + usage.cache_read_input_tokens, + usage.cache_read_tokens, + usage.cached_tokens, + ), + cacheWriteInputTokens: firstTokenCount( + promptDetails?.cache_write_tokens, + inputDetails?.cache_write_tokens, + usage.cache_creation_input_tokens, + usage.cache_write_input_tokens, + usage.cache_write_tokens, + ), + }) + : null + + const hasResponseMetadata = toNonEmptyString(data?.model) || normalizedUsage + if (!hasResponseMetadata) return currentMetadata || null + return mergeResponseMetadata(currentMetadata, { + selectedModel, + reportedModel: data?.model, + usage: normalizedUsage, + }) +} + +export function mergeClaudeResponseMetadata(currentMetadata, data, selectedModel) { + const usage = data?.message?.usage || data?.usage + const uncachedInputTokens = firstTokenCount(usage?.input_tokens) + const cacheReadInputTokens = firstTokenCount(usage?.cache_read_input_tokens) + const cacheWriteInputTokens = firstTokenCount(usage?.cache_creation_input_tokens) + const inputParts = [uncachedInputTokens, cacheReadInputTokens, cacheWriteInputTokens].filter( + (value) => value !== undefined, + ) + const currentInputTokens = toTokenCount(currentMetadata?.usage?.inputTokens) + const hasInputUsage = + Boolean(data?.message?.usage) || + (currentInputTokens === undefined && + (cacheReadInputTokens !== undefined || + cacheWriteInputTokens !== undefined || + (uncachedInputTokens !== undefined && uncachedInputTokens > 0))) + const normalizedUsage = usage + ? compactUsage({ + inputTokens: + hasInputUsage && inputParts.length > 0 + ? inputParts.reduce((total, value) => total + value, 0) + : undefined, + outputTokens: firstTokenCount(usage.output_tokens), + cacheReadInputTokens, + cacheWriteInputTokens, + }) + : null + + const reportedModel = data?.message?.model + const hasResponseMetadata = toNonEmptyString(reportedModel) || normalizedUsage + if (!hasResponseMetadata) return currentMetadata || null + return mergeResponseMetadata(currentMetadata, { + selectedModel, + reportedModel, + usage: normalizedUsage, + }) +} + +export function createRecordMetadata(session, metadata) { + return mergeResponseMetadata(metadata, { + selectedModel: metadata?.selectedModel || session?.modelName, + }) +} + +export function getRecordModel(metadata, fallbackModel = '') { + return ( + toNonEmptyString(metadata?.reportedModel) || + toNonEmptyString(metadata?.selectedModel) || + toNonEmptyString(fallbackModel) || + '' + ) +} + +export function formatTokenCount(value) { + const tokenCount = toTokenCount(value) + return tokenCount === undefined ? '' : tokenCount.toLocaleString() +} + +export function summarizeConversationUsage(records) { + const conversationRecords = Array.isArray(records) ? records : [] + const modelCounts = new Map() + const summary = { + totalTurns: conversationRecords.length, + reportedTurns: 0, + inputReportedTurns: 0, + outputReportedTurns: 0, + totalReportedTurns: 0, + cacheReadReportedTurns: 0, + cacheWriteReportedTurns: 0, + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + cacheReadInputTokens: 0, + cacheWriteInputTokens: 0, + models: [], + } + + for (const record of conversationRecords) { + const model = getRecordModel(record?.meta) + if (model) modelCounts.set(model, (modelCounts.get(model) || 0) + 1) + + const usage = compactUsage(record?.meta?.usage) + if (!usage) continue + summary.reportedTurns += 1 + if (usage.inputTokens !== undefined) { + summary.inputReportedTurns += 1 + summary.inputTokens += usage.inputTokens + } + if (usage.outputTokens !== undefined) { + summary.outputReportedTurns += 1 + summary.outputTokens += usage.outputTokens + } + if (usage.totalTokens !== undefined) { + summary.totalReportedTurns += 1 + summary.totalTokens += usage.totalTokens + } + if (usage.cacheReadInputTokens !== undefined) { + summary.cacheReadReportedTurns += 1 + summary.cacheReadInputTokens += usage.cacheReadInputTokens + } + if (usage.cacheWriteInputTokens !== undefined) { + summary.cacheWriteReportedTurns += 1 + summary.cacheWriteInputTokens += usage.cacheWriteInputTokens + } + } + + summary.models = Array.from(modelCounts, ([name, turns]) => ({ name, turns })) + return summary +} diff --git a/tests/unit/components/conversation-card-metadata.test.mjs b/tests/unit/components/conversation-card-metadata.test.mjs new file mode 100644 index 000000000..fc76e4b2c --- /dev/null +++ b/tests/unit/components/conversation-card-metadata.test.mjs @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + getCompletedAnswerMetadata, + getLastConversationRecord, +} from '../../../src/components/ConversationCard/session.mjs' + +test('getLastConversationRecord uses length-based indexing', () => { + const records = [{ question: 'Q1' }, { question: 'Q2' }] + + assert.equal(getLastConversationRecord(records), records[1]) + assert.equal(getLastConversationRecord([]), null) + assert.equal(getLastConversationRecord(null), null) +}) + +test('getCompletedAnswerMetadata reads retained response metadata', () => { + const meta = { selectedModel: 'model-a', usage: { inputTokens: 12 } } + + assert.equal( + getCompletedAnswerMetadata({ + message: { session: { conversationRecords: [{ meta }] } }, + restoredRetryAnswer: null, + partialAnswer: 'Answer', + retryRecord: null, + requestedModel: 'model-a', + fallbackModel: 'Model A', + }), + meta, + ) +}) + +test('getCompletedAnswerMetadata preserves metadata on duplicate done messages', () => { + assert.equal( + getCompletedAnswerMetadata({ + message: { done: true }, + restoredRetryAnswer: null, + partialAnswer: '', + retryRecord: null, + requestedModel: '', + fallbackModel: 'Model A', + }), + undefined, + ) +}) + +test('getCompletedAnswerMetadata records a model for retained partial answers', () => { + assert.deepEqual( + getCompletedAnswerMetadata({ + message: { done: true }, + restoredRetryAnswer: null, + partialAnswer: 'Partial', + retryRecord: null, + requestedModel: 'model-a', + fallbackModel: 'Model A', + }), + { selectedModel: 'model-a' }, + ) +}) + +test('getCompletedAnswerMetadata restores retry metadata', () => { + const meta = { selectedModel: 'old-model' } + + assert.equal( + getCompletedAnswerMetadata({ + message: { done: true }, + restoredRetryAnswer: 'Old answer', + partialAnswer: '', + retryRecord: { meta }, + requestedModel: 'new-model', + fallbackModel: 'New Model', + }), + meta, + ) +}) diff --git a/tests/unit/locales/usage-labels.test.mjs b/tests/unit/locales/usage-labels.test.mjs new file mode 100644 index 000000000..5beb9714f --- /dev/null +++ b/tests/unit/locales/usage-labels.test.mjs @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import { test } from 'node:test' + +const locales = [ + 'de', + 'en', + 'es', + 'fr', + 'id', + 'it', + 'ja', + 'ko', + 'pt', + 'ru', + 'tr', + 'zh-hans', + 'zh-hant', +] + +const usageKeys = [ + 'Input tokens', + 'Output tokens', + 'Total tokens', + 'Cached input tokens', + 'Cache write tokens', + 'Selected model', + 'Reported model', + 'Model', + 'Models', + 'Turns', + 'Reported usage', +] + +test('all locales include the model and token usage labels', async () => { + for (const locale of locales) { + const localeUrl = new URL(`../../../src/_locales/${locale}/main.json`, import.meta.url) + const translations = JSON.parse(await readFile(localeUrl, 'utf8')) + + for (const key of usageKeys) { + assert.equal( + typeof translations[key], + 'string', + `${locale} is missing the ${JSON.stringify(key)} translation`, + ) + assert.notEqual(translations[key].trim(), '') + } + } +}) diff --git a/tests/unit/services/apis/custom-api.test.mjs b/tests/unit/services/apis/custom-api.test.mjs index 57a6f7fde..3a1eee491 100644 --- a/tests/unit/services/apis/custom-api.test.mjs +++ b/tests/unit/services/apis/custom-api.test.mjs @@ -359,11 +359,18 @@ test('handles metadata-only SSE chunk without choices or response fields', async 'model', ) - assert.equal( - port.postedMessages.some((m) => m.done === false && m.answer === 'Hi'), - true, - ) - assert.deepEqual(session.conversationRecords.at(-1), { question: 'Q', answer: 'Hi' }) + const streamingAnswers = port.postedMessages + .filter((message) => message.done === false) + .map((message) => message.answer) + assert.deepEqual(streamingAnswers, ['Hi']) + assert.deepEqual(session.conversationRecords.at(-1), { + question: 'Q', + answer: 'Hi', + meta: { + selectedModel: 'model', + reportedModel: 'gpt-4', + }, + }) }) test('throws on non-ok response with JSON error body', async (t) => { diff --git a/tests/unit/services/apis/usage-abort-metadata.test.mjs b/tests/unit/services/apis/usage-abort-metadata.test.mjs new file mode 100644 index 000000000..c5b6e3952 --- /dev/null +++ b/tests/unit/services/apis/usage-abort-metadata.test.mjs @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict' +import { beforeEach, test } from 'node:test' +import { generateAnswersWithClaudeApi } from '../../../../src/services/apis/claude-api.mjs' +import { + generateAnswersWithOpenAICompatible, +} from '../../../../src/services/apis/openai-compatible-core.mjs' +import { createFakePort } from '../../helpers/port.mjs' + +const baseConfig = { + maxConversationContextLength: 3, + maxResponseTokenLength: 256, +} + +function sseData(data) { + return `data: ${JSON.stringify(data)}\n\n` +} + +function assertMetadataSessionFollowsStopAcknowledgement(port, session) { + const acknowledgementIndex = port.postedMessages.findIndex( + (message) => message.done === true && message.session === undefined, + ) + const sessionIndex = port.postedMessages.findIndex((message) => message.session === session) + + assert.notEqual(acknowledgementIndex, -1) + assert.notEqual(sessionIndex, -1) + assert.equal(port.postedMessages[sessionIndex].done, true) + assert.equal(acknowledgementIndex < sessionIndex, true) +} + +if (!globalThis.__TEST_BROWSER_SHIM__) { + globalThis.__TEST_BROWSER_SHIM__ = { + storage: {}, + clearStorage() { + this.storage = {} + }, + replaceStorage(values) { + this.storage = { ...values } + }, + } +} + +beforeEach(() => { + globalThis.__TEST_BROWSER_SHIM__.clearStorage() +}) + +test('OpenAI-compatible abort reposts retained metadata in a terminal session', async (t) => { + t.mock.method(console, 'debug', () => {}) + const session = { + aiName: 'Custom provider', + conversationRecords: [], + isRetry: false, + } + const port = createFakePort() + const encoder = new TextEncoder() + + t.mock.method(globalThis, 'fetch', async () => { + let readCount = 0 + return { + ok: true, + status: 200, + statusText: 'OK', + body: { + getReader() { + return { + async read() { + readCount += 1 + if (readCount === 1) { + return { + done: false, + value: encoder.encode( + sseData({ + model: 'reported-model', + choices: [{ delta: { content: 'Partial' } }], + usage: { + prompt_tokens: 10, + completion_tokens: 2, + total_tokens: 12, + }, + }), + ), + } + } + + port.emitMessage({ stop: true }) + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }, + } + }, + }, + } + }) + + await generateAnswersWithOpenAICompatible({ + port, + question: 'Hi', + session, + endpointType: 'chat', + requestUrl: 'https://proxy.example/v1/chat/completions', + model: 'selected-model', + apiKey: 'test-key', + config: baseConfig, + provider: 'compat', + }) + + assert.deepEqual(session.conversationRecords, [ + { + question: 'Hi', + answer: 'Partial', + meta: { + selectedModel: 'selected-model', + reportedModel: 'reported-model', + usage: { + inputTokens: 10, + outputTokens: 2, + totalTokens: 12, + }, + }, + }, + ]) + assertMetadataSessionFollowsStopAcknowledgement(port, session) + assert.deepEqual(port.listenerCounts(), { onMessage: 0, onDisconnect: 0 }) +}) + +test('aborted Claude stream reposts retained metadata with terminal session', async (t) => { + t.mock.method(console, 'debug', () => {}) + globalThis.__TEST_BROWSER_SHIM__.replaceStorage({ + customAnthropicApiUrl: 'https://api.anthropic.com', + anthropicApiKey: 'sk-ant-test', + maxConversationContextLength: 3, + maxResponseTokenLength: 256, + }) + const session = { + aiName: 'Anthropic (Claude Sonnet 5)', + modelName: 'claudeSonnet5Api', + conversationRecords: [], + isRetry: false, + } + const port = createFakePort() + const encoder = new TextEncoder() + + t.mock.method(globalThis, 'fetch', async () => { + let readCount = 0 + return { + ok: true, + status: 200, + statusText: 'OK', + body: { + getReader() { + return { + async read() { + readCount += 1 + if (readCount === 1) { + return { + done: false, + value: encoder.encode( + sseData({ + type: 'message_start', + message: { + model: 'claude-sonnet-5-20260801', + usage: { + input_tokens: 10, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 30, + output_tokens: 1, + }, + }, + }) + + sseData({ + type: 'content_block_delta', + delta: { type: 'text_delta', text: 'Partial' }, + }), + ), + } + } + + port.emitMessage({ stop: true }) + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }, + } + }, + }, + } + }) + + await generateAnswersWithClaudeApi(port, 'Hello', session) + + assert.deepEqual(session.conversationRecords, [ + { + question: 'Hello', + answer: 'Partial', + meta: { + selectedModel: 'claude-sonnet-5', + reportedModel: 'claude-sonnet-5-20260801', + usage: { + inputTokens: 60, + outputTokens: 1, + totalTokens: 61, + cacheReadInputTokens: 20, + cacheWriteInputTokens: 30, + }, + }, + }, + ]) + assertMetadataSessionFollowsStopAcknowledgement(port, session) + assert.deepEqual(port.listenerCounts(), { onMessage: 0, onDisconnect: 0 }) +}) diff --git a/tests/unit/services/apis/usage-records.test.mjs b/tests/unit/services/apis/usage-records.test.mjs new file mode 100644 index 000000000..aea99e33a --- /dev/null +++ b/tests/unit/services/apis/usage-records.test.mjs @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { pushRecord } from '../../../../src/services/apis/shared.mjs' + +test('pushRecord stores response metadata and falls back to the session model', () => { + const session = { + aiName: 'OpenAI (GPT-5.6)', + isRetry: false, + conversationRecords: [], + } + + pushRecord(session, 'Q', 'A', { + reportedModel: 'gpt-5.6-2026-08-01', + usage: { inputTokens: 10, outputTokens: 2 }, + }) + + assert.deepEqual(session.conversationRecords, [ + { + question: 'Q', + answer: 'A', + meta: { + selectedModel: 'OpenAI (GPT-5.6)', + reportedModel: 'gpt-5.6-2026-08-01', + usage: { inputTokens: 10, outputTokens: 2, totalTokens: 12 }, + }, + }, + ]) +}) + +test('pushRecord replaces stale answer metadata when retrying', () => { + const session = { + isRetry: true, + conversationRecords: [ + { + question: 'Q', + answer: 'Old', + meta: { + selectedModel: 'old-model', + usage: { inputTokens: 10, outputTokens: 2, totalTokens: 12 }, + }, + }, + ], + } + + pushRecord(session, 'Q', 'New', { + selectedModel: 'new-model', + usage: { inputTokens: 20, outputTokens: 4 }, + }) + + assert.deepEqual(session.conversationRecords, [ + { + question: 'Q', + answer: 'New', + meta: { + selectedModel: 'new-model', + usage: { inputTokens: 20, outputTokens: 4, totalTokens: 24 }, + }, + }, + ]) +}) + +test('pushRecord removes stale metadata when a retry has no model or usage metadata', () => { + const session = { + isRetry: true, + conversationRecords: [ + { + question: 'Q', + answer: 'Old', + meta: { selectedModel: 'old-model' }, + }, + ], + } + + pushRecord(session, 'Q', 'New') + + assert.deepEqual(session.conversationRecords, [{ question: 'Q', answer: 'New' }]) +}) diff --git a/tests/unit/services/apis/usage-streaming.test.mjs b/tests/unit/services/apis/usage-streaming.test.mjs new file mode 100644 index 000000000..a09b4690a --- /dev/null +++ b/tests/unit/services/apis/usage-streaming.test.mjs @@ -0,0 +1,311 @@ +import assert from 'node:assert/strict' +import { beforeEach, test } from 'node:test' +import { generateAnswersWithOpenAICompatible } from '../../../../src/services/apis/openai-compatible-core.mjs' +import { generateAnswersWithClaudeApi } from '../../../../src/services/apis/claude-api.mjs' +import { createFakePort } from '../../helpers/port.mjs' +import { createMockSseResponse } from '../../helpers/sse-response.mjs' + +const baseConfig = { + maxConversationContextLength: 3, + maxResponseTokenLength: 256, +} + +function sseData(data) { + return `data: ${JSON.stringify(data)}\n\n` +} + +if (!globalThis.__TEST_BROWSER_SHIM__) { + globalThis.__TEST_BROWSER_SHIM__ = { + storage: {}, + clearStorage() { + this.storage = {} + }, + replaceStorage(values) { + this.storage = { ...values } + }, + } +} + +beforeEach(() => { + globalThis.__TEST_BROWSER_SHIM__.clearStorage() +}) + +test('native OpenAI waits for the trailing usage block and requests streamed usage', async (t) => { + t.mock.method(console, 'debug', () => {}) + const session = { + aiName: 'OpenAI (GPT-5.6)', + conversationRecords: [], + isRetry: false, + } + const port = createFakePort() + let capturedInit + t.mock.method(globalThis, 'fetch', async (_input, init) => { + capturedInit = init + return createMockSseResponse([ + 'data: {"model":"gpt-5.6-2026-08-01","choices":[{"delta":{"content":"Hello"}}]}\n\n', + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + sseData({ + choices: [], + usage: { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 80 }, + }, + }), + ]) + }) + + await generateAnswersWithOpenAICompatible({ + port, + question: 'Hi', + session, + endpointType: 'chat', + requestUrl: 'https://api.openai.com/v1/chat/completions', + model: 'gpt-5.6', + apiKey: 'sk-test', + config: baseConfig, + provider: 'openai', + }) + + const body = JSON.parse(capturedInit.body) + assert.deepEqual(body.stream_options, { include_usage: true }) + assert.deepEqual(session.conversationRecords, [ + { + question: 'Hi', + answer: 'Hello', + meta: { + selectedModel: 'gpt-5.6', + reportedModel: 'gpt-5.6-2026-08-01', + usage: { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + cacheReadInputTokens: 80, + }, + }, + }, + ]) + assert.equal( + port.postedMessages.filter((message) => message.done === false).length, + 1, + ) + assert.deepEqual(port.listenerCounts(), { onMessage: 0, onDisconnect: 0 }) +}) + +test('OpenRouter is recognized by URL and records the routed model and cache usage', async (t) => { + t.mock.method(console, 'debug', () => {}) + const session = { + aiName: 'OpenRouter Auto', + conversationRecords: [], + isRetry: false, + } + const port = createFakePort() + let capturedInit + t.mock.method(globalThis, 'fetch', async (_input, init) => { + capturedInit = init + return createMockSseResponse([ + 'data: {"choices":[{"delta":{"content":"Routed"},"finish_reason":"stop"}]}\n\n', + sseData({ + model: 'anthropic/claude-sonnet-5', + choices: [{ delta: {}, finish_reason: 'stop' }], + usage: { + prompt_tokens: 90, + completion_tokens: 10, + total_tokens: 100, + prompt_tokens_details: { cached_tokens: 60, cache_write_tokens: 15 }, + }, + }), + ]) + }) + + await generateAnswersWithOpenAICompatible({ + port, + question: 'Route this', + session, + endpointType: 'chat', + requestUrl: 'https://openrouter.ai/api/v1/chat/completions', + model: 'openrouter/auto', + apiKey: 'or-test', + config: baseConfig, + provider: 'compat', + }) + + assert.equal(Object.hasOwn(JSON.parse(capturedInit.body), 'stream_options'), false) + const lastRecord = session.conversationRecords[session.conversationRecords.length - 1] + assert.deepEqual(lastRecord.meta, { + selectedModel: 'openrouter/auto', + reportedModel: 'anthropic/claude-sonnet-5', + usage: { + inputTokens: 90, + outputTokens: 10, + totalTokens: 100, + cacheReadInputTokens: 60, + cacheWriteInputTokens: 15, + }, + }) +}) + +test('Anthropic streaming stores cumulative output and full cache-aware input usage', async (t) => { + t.mock.method(console, 'debug', () => {}) + globalThis.__TEST_BROWSER_SHIM__.replaceStorage({ + customAnthropicApiUrl: 'https://api.anthropic.com', + anthropicApiKey: 'sk-ant-test', + maxConversationContextLength: 3, + maxResponseTokenLength: 256, + }) + const session = { + aiName: 'Anthropic (Claude Sonnet 5)', + modelName: 'claudeSonnet5Api', + conversationRecords: [], + isRetry: false, + } + const port = createFakePort() + t.mock.method(globalThis, 'fetch', async () => + createMockSseResponse([ + sseData({ + type: 'message_start', + message: { + model: 'claude-sonnet-5-20260801', + usage: { + input_tokens: 10, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 30, + output_tokens: 1, + }, + }, + }), + 'data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"Claude"}}\n\n', + 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":0,"output_tokens":6}}\n\n', + 'data: {"type":"message_stop"}\n\n', + ]), + ) + + await generateAnswersWithClaudeApi(port, 'Hello', session) + + assert.deepEqual(session.conversationRecords, [ + { + question: 'Hello', + answer: 'Claude', + meta: { + selectedModel: 'claude-sonnet-5', + reportedModel: 'claude-sonnet-5-20260801', + usage: { + inputTokens: 60, + outputTokens: 6, + totalTokens: 66, + cacheReadInputTokens: 20, + cacheWriteInputTokens: 30, + }, + }, + }, + ]) +}) + +test('OpenAI preserves a completed answer when the stream breaks before usage arrives', async (t) => { + t.mock.method(console, 'debug', () => {}) + const session = { + aiName: 'OpenAI (GPT-5.6)', + conversationRecords: [], + isRetry: false, + } + const port = createFakePort() + const encoder = new TextEncoder() + t.mock.method(globalThis, 'fetch', async () => { + let readCount = 0 + return { + ok: true, + status: 200, + statusText: 'OK', + body: { + getReader() { + return { + async read() { + readCount += 1 + if (readCount === 1) { + return { + done: false, + value: encoder.encode( + 'data: {"model":"gpt-5.6","choices":[{"delta":{"content":"Complete"}}]}\n\n', + ), + } + } + if (readCount === 2) { + return { + done: false, + value: encoder.encode( + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + ), + } + } + throw new Error('stream interrupted') + }, + } + }, + }, + } + }) + + await generateAnswersWithOpenAICompatible({ + port, + question: 'Hi', + session, + endpointType: 'chat', + requestUrl: 'https://api.openai.com/v1/chat/completions', + model: 'gpt-5.6', + apiKey: 'sk-test', + config: baseConfig, + provider: 'openai', + }) + + assert.deepEqual(session.conversationRecords, [ + { + question: 'Hi', + answer: 'Complete', + meta: { + selectedModel: 'gpt-5.6', + reportedModel: 'gpt-5.6', + }, + }, + ]) + const lastMessage = port.postedMessages[port.postedMessages.length - 1] + assert.equal(lastMessage.done, true) +}) + +test('a custom OpenAI-compatible endpoint is not forced to accept stream_options', async (t) => { + t.mock.method(console, 'debug', () => {}) + const session = { + aiName: 'Custom OpenAI endpoint', + conversationRecords: [], + isRetry: false, + } + const port = createFakePort() + let capturedInit + t.mock.method(globalThis, 'fetch', async (_input, init) => { + capturedInit = init + return createMockSseResponse([ + 'data: {"choices":[{"delta":{"content":"Compatible"},"finish_reason":"stop"}]}\n\n', + ]) + }) + + await generateAnswersWithOpenAICompatible({ + port, + question: 'Hi', + session, + endpointType: 'chat', + requestUrl: 'https://proxy.example/v1/chat/completions', + model: 'custom-model', + apiKey: 'sk-test', + config: baseConfig, + provider: 'openai', + }) + + assert.equal(Object.hasOwn(JSON.parse(capturedInit.body), 'stream_options'), false) + assert.deepEqual(session.conversationRecords, [ + { + question: 'Hi', + answer: 'Compatible', + meta: { selectedModel: 'Custom OpenAI endpoint' }, + }, + ]) +}) diff --git a/tests/unit/utils/usage-metadata.test.mjs b/tests/unit/utils/usage-metadata.test.mjs new file mode 100644 index 000000000..ab0703f9c --- /dev/null +++ b/tests/unit/utils/usage-metadata.test.mjs @@ -0,0 +1,192 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + createRecordMetadata, + mergeClaudeResponseMetadata, + mergeOpenAIResponseMetadata, + summarizeConversationUsage, +} from '../../../src/utils/usage-metadata.mjs' + +test('OpenAI metadata merges the reported model and final usage block', () => { + let metadata = mergeOpenAIResponseMetadata( + null, + { + model: 'gpt-5.6-2026-08-01', + choices: [{ delta: { content: 'Hello' } }], + }, + 'gpt-5.6', + ) + + metadata = mergeOpenAIResponseMetadata( + metadata, + { + choices: [], + usage: { + prompt_tokens: 120, + completion_tokens: 30, + total_tokens: 150, + prompt_tokens_details: { + cached_tokens: 80, + cache_write_tokens: 20, + }, + }, + }, + 'gpt-5.6', + ) + + assert.deepEqual(metadata, { + selectedModel: 'gpt-5.6', + reportedModel: 'gpt-5.6-2026-08-01', + usage: { + inputTokens: 120, + outputTokens: 30, + totalTokens: 150, + cacheReadInputTokens: 80, + cacheWriteInputTokens: 20, + }, + }) +}) + +test('OpenAI metadata distinguishes zero cached tokens from an unavailable field', () => { + const zeroCacheMetadata = mergeOpenAIResponseMetadata( + null, + { + usage: { + prompt_tokens: 10, + completion_tokens: 2, + prompt_tokens_details: { cached_tokens: 0 }, + }, + }, + 'gpt-test', + ) + const unavailableCacheMetadata = mergeOpenAIResponseMetadata( + null, + { + usage: { + prompt_tokens: 10, + completion_tokens: 2, + }, + }, + 'gpt-test', + ) + + assert.equal(zeroCacheMetadata.usage.cacheReadInputTokens, 0) + assert.equal(Object.hasOwn(unavailableCacheMetadata.usage, 'cacheReadInputTokens'), false) +}) + +test('Claude metadata treats cache reads and writes as parts of the full input', () => { + let metadata = mergeClaudeResponseMetadata( + null, + { + type: 'message_start', + message: { + model: 'claude-sonnet-5-20260801', + usage: { + input_tokens: 10, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 30, + output_tokens: 1, + }, + }, + }, + 'claude-sonnet-5', + ) + + metadata = mergeClaudeResponseMetadata( + metadata, + { + type: 'message_delta', + usage: { + input_tokens: 0, + output_tokens: 5, + }, + }, + 'claude-sonnet-5', + ) + + assert.deepEqual(metadata, { + selectedModel: 'claude-sonnet-5', + reportedModel: 'claude-sonnet-5-20260801', + usage: { + inputTokens: 60, + outputTokens: 5, + totalTokens: 65, + cacheReadInputTokens: 20, + cacheWriteInputTokens: 30, + }, + }) +}) + +test('record metadata falls back to the stable session model identifier', () => { + assert.deepEqual( + createRecordMetadata( + { + modelName: 'chatgptApi5_6', + aiName: 'OpenAI (GPT-5.6)', + }, + null, + ), + { selectedModel: 'chatgptApi5_6' }, + ) + assert.equal(createRecordMetadata({ aiName: 'OpenAI (GPT-5.6)' }, null), null) +}) + +test('conversation summary totals only complete reported usage and preserves model history', () => { + const summary = summarizeConversationUsage([ + { + question: 'one', + answer: 'a', + meta: { + selectedModel: 'router/auto', + reportedModel: 'model-a', + usage: { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + cacheReadInputTokens: 40, + }, + }, + }, + { + question: 'two', + answer: 'b', + meta: { + selectedModel: 'model-b', + usage: { + inputTokens: 50, + outputTokens: 10, + cacheReadInputTokens: 0, + cacheWriteInputTokens: 15, + }, + }, + }, + { + question: 'three', + answer: 'c', + meta: { + selectedModel: 'model-b', + usage: { outputTokens: 4 }, + }, + }, + { question: 'legacy', answer: 'record' }, + ]) + + assert.deepEqual(summary, { + totalTurns: 4, + reportedTurns: 3, + inputReportedTurns: 2, + outputReportedTurns: 3, + totalReportedTurns: 2, + cacheReadReportedTurns: 2, + cacheWriteReportedTurns: 1, + inputTokens: 150, + outputTokens: 34, + totalTokens: 180, + cacheReadInputTokens: 40, + cacheWriteInputTokens: 15, + models: [ + { name: 'model-a', turns: 1 }, + { name: 'model-b', turns: 2 }, + ], + }) +})