diff --git a/packages/storage/src/__tests__/public-entrypoints.test.ts b/packages/storage/src/__tests__/public-entrypoints.test.ts index c177133c03..d3d185cb93 100644 --- a/packages/storage/src/__tests__/public-entrypoints.test.ts +++ b/packages/storage/src/__tests__/public-entrypoints.test.ts @@ -66,7 +66,6 @@ const SQLITE_BACKED_ENTRYPOINTS = [ './session-bundle-policy', './session-copy-cleanup', './session-store', - './settings-store', './shell-run-authority', './shell-run-store', './sqlite-runtime-store', diff --git a/packages/storage/src/__tests__/settings-store-usage.test.ts b/packages/storage/src/__tests__/settings-store-usage.test.ts deleted file mode 100644 index 0d8f43a107..0000000000 --- a/packages/storage/src/__tests__/settings-store-usage.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { strict as assert } from 'node:assert'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { DatabaseSync } from 'node:sqlite'; -import { describe, it } from 'node:test'; -import type { StoredMessage } from '@maka/core/session'; -import { createSessionStore } from '../session-store.js'; -import { createSettingsStore } from '../settings-store.js'; - -async function seedSession(workspaceRoot: string, messages: StoredMessage[]): Promise { - const sessions = createSessionStore(workspaceRoot); - try { - const header = await sessions.create({ - cwd: '/tmp/maka-workspace', - llmConnectionSlug: 'anthropic', - model: 'claude-sonnet-4', - permissionMode: 'ask', - name: 'Usage fixture', - }); - await sessions.appendMessages(header.id, messages); - return header.id; - } finally { - await sessions.close?.(); - } -} - -describe('SettingsStore.usageStats request logs', () => { - it('does not migrate session metadata when operational state is rejected', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-settings-usage-rejected-')); - const databasePath = join(workspaceRoot, 'runtime.sqlite'); - try { - const sessions = createSessionStore(workspaceRoot); - await sessions.close?.(); - - const database = new DatabaseSync(databasePath); - database - .prepare(`UPDATE session_metadata_schema SET version = 21 WHERE scope = 'session_metadata'`) - .run(); - database - .prepare( - `INSERT INTO operational_schema_migrations(scope, version, applied_at) - VALUES ('future_scope', 1, 0)`, - ) - .run(); - database.close(); - - await assert.rejects( - () => createSettingsStore(workspaceRoot).usageStats('all'), - /Operational schema future_scope is unknown to this Maka build/, - ); - - const preserved = new DatabaseSync(databasePath, { readOnly: true }); - try { - assert.equal( - ( - preserved - .prepare( - `SELECT version FROM session_metadata_schema WHERE scope = 'session_metadata'`, - ) - .get() as { version: number } - ).version, - 21, - ); - } finally { - preserved.close(); - } - } finally { - await rm(workspaceRoot, { recursive: true, force: true }); - } - }); - - it('uses canonical SQLite metadata for transcript-marker sessions', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-settings-usage-sqlite-')); - const sessions = createSessionStore(workspaceRoot); - try { - const header = await sessions.create({ - cwd: '/tmp/maka-workspace', - llmConnectionSlug: 'sqlite-provider', - model: 'sqlite-default-model', - permissionMode: 'ask', - }); - await sessions.appendMessages(header.id, [ - { - type: 'assistant', - id: 'assistant-1', - turnId: 'turn-1', - ts: 10, - text: 'tracked', - modelId: 'sqlite-runtime-model', - }, - { - type: 'token_usage', - id: 'usage-1', - turnId: 'turn-1', - ts: 20, - input: 8, - output: 2, - }, - ]); - - const stats = await createSettingsStore(workspaceRoot).usageStats('all'); - - assert.equal(stats.summary.totalTokens, 10); - assert.equal(stats.logs[0]?.provider, 'sqlite-provider'); - assert.equal(stats.logs[0]?.model, 'sqlite-runtime-model'); - } finally { - await sessions.close?.(); - await rm(workspaceRoot, { recursive: true, force: true }); - } - }); - - it('includes tool invocation rows without inflating model usage totals', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-settings-usage-')); - try { - const sessionId = await seedSession(workspaceRoot, [ - { - type: 'assistant', - id: 'assistant-1', - turnId: 'turn-1', - ts: 10, - text: 'I will inspect it.', - modelId: 'claude-sonnet-4-runtime', - }, - { - type: 'tool_call', - id: 'tool-1', - turnId: 'turn-1', - ts: 11, - toolName: 'Bash', - displayName: '终端', - args: { cmd: 'pwd' }, - }, - { - type: 'tool_result', - id: 'tool-result-1', - turnId: 'turn-1', - ts: 15, - toolUseId: 'tool-1', - isError: true, - durationMs: 37, - content: { kind: 'text', text: 'failed' }, - }, - { - type: 'token_usage', - id: 'usage-1', - turnId: 'turn-1', - ts: 20, - input: 120, - output: 30, - cacheMissInput: 105, - cacheRead: 10, - cacheCreation: 5, - reasoning: 4, - costUsd: 0.01, - }, - ]); - - const stats = await createSettingsStore(workspaceRoot).usageStats('all'); - - assert.equal(stats.summary.totalRequests, 1, 'summary counts model requests only'); - assert.equal(stats.summary.totalTokens, 150); - assert.equal(stats.summary.totalCostUsd, 0.01); - assert.equal(stats.summary.cacheMiss, 105); - assert.equal(stats.summary.cacheRead, 10); - assert.equal(stats.summary.cacheCreation, 5); - assert.equal(stats.summary.reasoning, 4); - assert.equal(stats.byProvider.length, 1, 'provider aggregates remain model-only'); - assert.equal(stats.byModel.length, 1, 'model aggregates remain model-only'); - - const modelLog = stats.logs.find((log) => log.kind === 'model'); - assert.ok(modelLog); - assert.equal(modelLog.sessionId, sessionId); - assert.equal(modelLog.turnId, 'turn-1'); - assert.equal(modelLog.model, 'claude-sonnet-4-runtime'); - assert.equal(modelLog.inputTokens, 120); - assert.equal(modelLog.outputTokens, 30); - assert.equal(modelLog.cacheMiss, 105); - assert.equal(modelLog.cacheRead, 10); - assert.equal(modelLog.cacheCreation, 5); - assert.equal(modelLog.reasoning, 4); - - const toolLog = stats.logs.find((log) => log.kind === 'tool'); - assert.ok(toolLog); - assert.equal(toolLog.id, 'tool:tool-1'); - assert.equal(toolLog.sessionId, sessionId); - assert.equal(toolLog.turnId, 'turn-1'); - assert.equal(toolLog.provider, 'anthropic'); - assert.equal(toolLog.model, 'claude-sonnet-4'); - assert.equal(toolLog.toolName, '终端'); - assert.equal(toolLog.inputTokens, 0); - assert.equal(toolLog.outputTokens, 0); - assert.equal(toolLog.latencyMs, 37); - assert.equal(toolLog.status, 'error'); - } finally { - await rm(workspaceRoot, { recursive: true, force: true }); - } - }); - - it('merges tool stats by tool name across sessions instead of one row per session', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-settings-usage-bytool-')); - try { - // Two sessions each call Bash; the second also calls Read. Tool-call ids - // are only unique within a session, so both sessions reuse `bash`/`read` - // ids to prove result matching stays session-scoped after the merge. - const bashTurn = (sessionId: string, error: boolean, duration: number): StoredMessage[] => [ - { - type: 'tool_call', - id: 'bash', - turnId: 't1', - ts: 10, - toolName: 'Bash', - displayName: '终端', - args: {}, - }, - { - type: 'tool_result', - id: 'bash-result', - turnId: 't1', - ts: 10 + duration, - toolUseId: 'bash', - isError: error, - durationMs: duration, - content: { kind: 'text', text: error ? 'failed' : 'ok' }, - }, - ]; - - await seedSession(workspaceRoot, [ - ...bashTurn('session-a', false, 20), - { - type: 'tool_call', - id: 'read', - turnId: 't1', - ts: 12, - toolName: 'Read', - displayName: '读取', - args: {}, - }, - { - type: 'tool_result', - id: 'read-result', - turnId: 't1', - ts: 40, - toolUseId: 'read', - isError: false, - durationMs: 28, - content: { kind: 'text', text: 'ok' }, - }, - ]); - await seedSession(workspaceRoot, [...bashTurn('session-b', true, 40)]); - - const stats = await createSettingsStore(workspaceRoot).usageStats('all'); - - // One row per tool name — never a duplicate Bash row per session. - assert.equal(stats.byTool.length, 2, 'byTool must have one row per unique tool'); - const bash = stats.byTool.find((row) => row.tool === 'Bash'); - assert.ok(bash, 'a single merged Bash row must exist'); - assert.equal(bash.calls, 2, 'Bash calls merge across sessions'); - assert.equal(bash.success, 1, 'the successful Bash call is counted'); - assert.equal(bash.errors, 1, 'the failed Bash call is counted'); - assert.equal(bash.avgDurationMs, 30, 'Bash duration averages (20 + 40) / 2 across sessions'); - - const read = stats.byTool.find((row) => row.tool === 'Read'); - assert.ok(read); - assert.equal(read.calls, 1); - assert.equal(read.avgDurationMs, 28); - - // Rows are ordered by call count desc so the busiest tool leads. - assert.deepEqual( - stats.byTool.map((row) => row.tool), - ['Bash', 'Read'], - ); - } finally { - await rm(workspaceRoot, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/storage/src/settings-store.ts b/packages/storage/src/settings-store.ts index 20f369374d..39b0377d1b 100644 --- a/packages/storage/src/settings-store.ts +++ b/packages/storage/src/settings-store.ts @@ -19,17 +19,10 @@ import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; -import type { - AppSettings, - SettingsTestResult, - UpdateAppSettingsInput, - UsageRange, - UsageStats, -} from '@maka/core/settings'; +import type { AppSettings, SettingsTestResult, UpdateAppSettingsInput } from '@maka/core/settings'; import type { OnboardingMilestone, OnboardingMilestoneId } from '@maka/core/onboarding'; import { createDefaultSettings, mergeSettings, normalizeSettings } from '@maka/core/settings'; import { sanitizeOnboardingMilestones } from '@maka/core/onboarding'; -import { readUsageStats } from './usage-stats-store.js'; /** * A conditional write's patch, either fixed or derived from the state the @@ -53,7 +46,6 @@ export interface SettingsStore { patch: ConditionalSettingsPatch, ): Promise<{ applied: boolean; settings: AppSettings }>; testNetworkProxy(): Promise; - usageStats(range?: UsageRange): Promise; /** * PR110b: upsert a single onboarding milestone. Caller passes the * desired terminal status; the store stamps `Date.now()` so the @@ -214,10 +206,6 @@ class FileSettingsStore implements SettingsStore { }; } - async usageStats(range: UsageRange = '24h'): Promise { - return readUsageStats(this.workspaceRoot, range); - } - private async write(settings: AppSettings): Promise { await mkdir(dirname(this.settingsPath), { recursive: true }); const tempPath = `${this.settingsPath}.${process.pid}.${Date.now()}.tmp`; diff --git a/packages/storage/src/usage-stats-store.ts b/packages/storage/src/usage-stats-store.ts deleted file mode 100644 index ecd34981e7..0000000000 --- a/packages/storage/src/usage-stats-store.ts +++ /dev/null @@ -1,391 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { join } from 'node:path'; -import type { UsageRange, UsageStats } from '@maka/core/settings'; -import { legacyUsageProvenance } from '@maka/core/usage-ledger-merge'; -import type { SessionHeader } from '@maka/core/session'; -import { - acquireOperationalStateDatabase, - OPERATIONAL_STATE_DATABASE_NAME, -} from './operational-state-store.js'; -import { createSqliteSessionMetadataStore } from './sqlite-session-metadata-store.js'; - -type UsageSessionHeader = Pick; - -type UsageAssistantMessage = { - type: 'assistant'; - turnId: string; - modelId: string; -}; - -type UsageTokenMessage = { - type: 'token_usage'; - id: string; - turnId: string; - ts: number; - input: number; - output: number; - cacheMissInput?: number; - cacheRead?: number; - cacheCreation?: number; - reasoning?: number; - costUsd?: number; -}; - -type UsageToolCallMessage = { - type: 'tool_call'; - id: string; - turnId: string; - ts: number; - toolName: string; - displayName?: string; -}; - -type UsageToolResultMessage = { - type: 'tool_result'; - turnId: string; - ts: number; - toolUseId: string; - isError: boolean; - durationMs?: number; -}; - -type UsageMessage = - | UsageAssistantMessage - | UsageTokenMessage - | UsageToolCallMessage - | UsageToolResultMessage; - -export async function readUsageStats( - workspaceRoot: string, - range: UsageRange, -): Promise { - const since = rangeToSince(range); - const sessions = await readStoredSessions(workspaceRoot); - const modelLogs = sessions.flatMap(({ header, messages }) => { - const assistantByTurn = new Map( - messages - .filter((message) => message.type === 'assistant') - .map((message) => [message.turnId, message.modelId]), - ); - return messages - .filter((message): message is UsageTokenMessage => message.type === 'token_usage') - .filter((message) => !since || message.ts >= since) - .map((message) => ({ - id: message.id, - ts: message.ts, - kind: 'model' as const, - sessionId: header.id, - sessionName: header.name, - turnId: message.turnId, - provider: header.llmConnectionSlug, - model: assistantByTurn.get(message.turnId) ?? header.model, - inputTokens: message.input, - outputTokens: message.output, - cacheMiss: message.cacheMissInput, - cacheRead: message.cacheRead, - cacheCreation: message.cacheCreation, - reasoning: message.reasoning, - costUsd: message.costUsd, - status: 'success' as const, - })); - }); - - const toolRows = aggregateToolStats(sessions, since); - const toolLogs = sessions.flatMap(({ header, messages }) => - toolLogRowsFromMessages(header, messages, since), - ); - const logs = [...modelLogs, ...toolLogs].sort((a, b) => b.ts - a.ts); - const totalInput = sum(modelLogs.map((log) => log.inputTokens)); - const totalOutput = sum(modelLogs.map((log) => log.outputTokens)); - const cacheMiss = sum(modelLogs.map((log) => log.cacheMiss ?? 0)); - const cacheRead = sum(modelLogs.map((log) => log.cacheRead ?? 0)); - const cacheCreation = sum(modelLogs.map((log) => log.cacheCreation ?? 0)); - const reasoning = sum(modelLogs.map((log) => log.reasoning ?? 0)); - return { - summary: { - totalRequests: modelLogs.length, - totalCostUsd: sum(modelLogs.map((log) => log.costUsd ?? 0)), - totalTokens: totalInput + totalOutput, - inputTokens: totalInput, - outputTokens: totalOutput, - cacheTokens: cacheRead + cacheCreation, - cacheMiss, - cacheRead, - cacheCreation, - reasoning, - }, - logs, - byProvider: aggregateBy(modelLogs, 'provider'), - byModel: aggregateBy(modelLogs, 'model'), - byTool: toolRows, - pricing: [], - // This store reads the frozen pre-canonical session records, so its rows are - // all legacy: cost is present but was never qualified with a cost basis. - provenance: legacyUsageProvenance(modelLogs.length), - }; -} - -async function readStoredSessions( - workspaceRoot: string, -): Promise> { - const databaseLease = acquireOperationalStateDatabase(workspaceRoot); - const metadata = createSqliteSessionMetadataStore( - join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), - { databaseLease }, - ); - try { - const sessions: Array<{ header: UsageSessionHeader; messages: UsageMessage[] }> = []; - // Usage is a durable cost aggregate, so it counts every Session that spent - // tokens, including reserved roles and rows a catalog would hide. - for (const { header } of await metadata.list(undefined, 'all')) { - const messages = (await metadata.readMessages(header.id)).flatMap((value) => { - const message = normalizeUsageMessage(value); - return message ? [message] : []; - }); - sessions.push({ - header: { - id: header.id, - name: header.name, - llmConnectionSlug: header.llmConnectionSlug, - model: header.model, - }, - messages, - }); - } - return sessions; - } catch { - return []; - } finally { - metadata.close(); - } -} - -function normalizeUsageSessionHeader(value: unknown, sessionId: string): UsageSessionHeader | null { - if (!isRecord(value)) return null; - if (value.id !== sessionId) return null; - if (typeof value.name !== 'string') return null; - if (typeof value.llmConnectionSlug !== 'string') return null; - if (typeof value.model !== 'string') return null; - return { - id: value.id, - name: value.name, - llmConnectionSlug: value.llmConnectionSlug, - model: value.model, - }; -} - -function normalizeUsageMessage(value: unknown): UsageMessage | null { - if (!isRecord(value)) return null; - switch (value.type) { - case 'assistant': - if (typeof value.turnId !== 'string') return null; - if (typeof value.modelId !== 'string') return null; - return { type: 'assistant', turnId: value.turnId, modelId: value.modelId }; - case 'token_usage': - if (typeof value.id !== 'string') return null; - if (typeof value.turnId !== 'string') return null; - if (!isFiniteNumber(value.ts)) return null; - if (!isFiniteNumber(value.input)) return null; - if (!isFiniteNumber(value.output)) return null; - if (!isOptionalFiniteNumber(value.cacheMissInput)) return null; - if (!isOptionalFiniteNumber(value.cacheRead)) return null; - if (!isOptionalFiniteNumber(value.cacheCreation)) return null; - if (!isOptionalFiniteNumber(value.reasoning)) return null; - if (!isOptionalFiniteNumber(value.costUsd)) return null; - return { - type: 'token_usage', - id: value.id, - turnId: value.turnId, - ts: value.ts, - input: value.input, - output: value.output, - cacheMissInput: value.cacheMissInput, - cacheRead: value.cacheRead, - cacheCreation: value.cacheCreation, - reasoning: value.reasoning, - costUsd: value.costUsd, - }; - case 'tool_call': - if (typeof value.id !== 'string') return null; - if (typeof value.turnId !== 'string') return null; - if (!isFiniteNumber(value.ts)) return null; - if (typeof value.toolName !== 'string') return null; - if (value.displayName !== undefined && typeof value.displayName !== 'string') return null; - return { - type: 'tool_call', - id: value.id, - turnId: value.turnId, - ts: value.ts, - toolName: value.toolName, - displayName: value.displayName, - }; - case 'tool_result': - if (typeof value.turnId !== 'string') return null; - if (!isFiniteNumber(value.ts)) return null; - if (typeof value.toolUseId !== 'string') return null; - if (typeof value.isError !== 'boolean') return null; - if (!isOptionalFiniteNumber(value.durationMs)) return null; - return { - type: 'tool_result', - turnId: value.turnId, - ts: value.ts, - toolUseId: value.toolUseId, - isError: value.isError, - durationMs: value.durationMs, - }; - default: - return null; - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function isFiniteNumber(value: unknown): value is number { - return typeof value === 'number' && Number.isFinite(value); -} - -function isOptionalFiniteNumber(value: unknown): value is number | undefined { - return value === undefined || isFiniteNumber(value); -} - -function rangeToSince(range: UsageRange): number | null { - const now = Date.now(); - switch (range) { - case '24h': - return now - 24 * 60 * 60 * 1000; - case '7d': - return now - 7 * 24 * 60 * 60 * 1000; - case '30d': - return now - 30 * 24 * 60 * 60 * 1000; - case 'all': - return null; - } -} - -function aggregateBy(logs: UsageStats['logs'], key: 'provider' | 'model') { - const rows = new Map(); - for (const log of logs) { - const id = log[key]; - const current = rows.get(id) ?? { requests: 0, tokens: 0, costUsd: 0 }; - current.requests += 1; - current.tokens += log.inputTokens + log.outputTokens; - current.costUsd += log.costUsd ?? 0; - rows.set(id, current); - } - return [...rows.entries()] - .map(([id, row]) => ({ [key]: id, ...row })) - .sort((a, b) => b.requests - a.requests) as never; -} - -// Aggregate tool usage by tool name across EVERY session so 工具统计 shows one row -// per tool (not one row per tool-per-session, which repeated the same tool name). -// tool_call.id ↔ tool_result.toolUseId matching stays scoped to each session's -// messages — ids are only unique within a session — while the counts, failures, -// and durations merge into a single global row keyed by tool name. -function aggregateToolStats( - sessions: Array<{ messages: UsageMessage[] }>, - since: number | null, -): UsageStats['byTool'] { - const rows = new Map< - string, - { calls: number; success: number; errors: number; totalDuration: number; durationCount: number } - >(); - for (const { messages } of sessions) { - const results = new Map( - messages - .filter((message): message is UsageToolResultMessage => message.type === 'tool_result') - .map((message) => [message.toolUseId, message]), - ); - const calls = messages.filter( - (message): message is UsageToolCallMessage => message.type === 'tool_call', - ); - for (const call of calls) { - if (since && call.ts < since) continue; - const result = results.get(call.id); - const current = rows.get(call.toolName) ?? { - calls: 0, - success: 0, - errors: 0, - totalDuration: 0, - durationCount: 0, - }; - current.calls += 1; - if (result?.isError) current.errors += 1; - else current.success += 1; - if (result?.durationMs !== undefined) { - current.totalDuration += result.durationMs; - current.durationCount += 1; - } - rows.set(call.toolName, current); - } - } - return [...rows.entries()] - .map(([tool, row]) => ({ - tool, - calls: row.calls, - success: row.success, - errors: row.errors, - avgDurationMs: row.durationCount ? Math.round(row.totalDuration / row.durationCount) : 0, - })) - .sort((a, b) => b.calls - a.calls || a.tool.localeCompare(b.tool)); -} - -function toolLogRowsFromMessages( - header: UsageSessionHeader, - messages: UsageMessage[], - since: number | null, -): UsageStats['logs'] { - const calls = messages.filter( - (message): message is UsageToolCallMessage => message.type === 'tool_call', - ); - const results = new Map( - messages - .filter((message): message is UsageToolResultMessage => message.type === 'tool_result') - .map((message) => [message.toolUseId, message]), - ); - return calls - .filter((call) => !since || call.ts >= since) - .map((call) => { - const result = results.get(call.id); - const ts = result?.ts ?? call.ts; - return { - id: `tool:${call.id}`, - ts, - kind: 'tool' as const, - sessionId: header.id, - sessionName: header.name, - turnId: call.turnId, - provider: header.llmConnectionSlug, - model: header.model, - toolName: call.displayName ?? call.toolName, - inputTokens: 0, - outputTokens: 0, - latencyMs: result?.durationMs, - status: result?.isError ? ('error' as const) : ('success' as const), - }; - }); -} - -function sum(values: number[]): number { - return values.reduce((total, value) => total + value, 0); -}