diff --git a/src/content-utils.ts b/src/content-utils.ts index 5ca31ae9..f3717194 100644 --- a/src/content-utils.ts +++ b/src/content-utils.ts @@ -24,3 +24,34 @@ export function normalizeContentBlocks() + async function resolveCanonicalProjectPath(cwd: string): Promise<{ path: string; isWorktree: boolean }> { + const cached = canonicalPathCache.get(cwd) + if (cached) return cached + const result = await resolveCanonicalProjectPathUncached(cwd) + canonicalPathCache.set(cwd, result) + return result +} + +async function resolveCanonicalProjectPathUncached(cwd: string): Promise<{ path: string; isWorktree: boolean }> { const trimmed = cwd.trim() if (!trimmed) return { path: cwd, isWorktree: false } @@ -1261,7 +1278,7 @@ export function collectToolResultMeta(entry: JournalEntry, map: Map)['aiTitle'] - if (typeof t === 'string' && t.trim()) meta.title = t.trim().slice(0, 200) + if (typeof t === 'string' && t.trim()) meta.title = flatString(t.trim().slice(0, 200)) } else if (entry.type === 'pr-link') { const url = (entry as Record)['prUrl'] if (typeof url === 'string' && url && !meta.prLinks.includes(url)) meta.prLinks.push(url) @@ -1900,7 +1917,7 @@ export async function readAgentType(filePath: string): Promise')) continue inputChars += msg.content.length - pendingUserMessage = msg.content.slice(0, 500) + pendingUserMessage = flatSlice(msg.content, 500) } if (msg.role === 'bot') { const msgTools = extractToolNames(msg.content) @@ -296,7 +300,7 @@ function parseModernExecution(data: KiroModernExecution, sourcePath: string, see if (directInput) { inputChars += directInput.length - pendingUserMessage = directInput.slice(0, 500) + pendingUserMessage = flatSlice(directInput, 500) } if (directOutput) { @@ -328,7 +332,7 @@ function parseModernExecution(data: KiroModernExecution, sourcePath: string, see if (role === 'human' || role === 'user') { if (!text) continue inputChars += text.length - pendingUserMessage = text.slice(0, 500) + pendingUserMessage = flatSlice(text, 500) } else if (role === 'bot' || role === 'assistant' || role === 'ai' || role === 'model') { if (text) outputChars += text.length if (text || tools.length > 0) hasOutputActivity = true @@ -506,6 +510,7 @@ function parseCliSession(meta: KiroCliSessionMeta, entries: KiroCliEntry[], seen userMessage: pendingUserMessage, sessionId, project, + ...(meta.cwd ? { projectPath: meta.cwd } : {}), }) turnIndex++ } @@ -526,7 +531,7 @@ function parseCliSession(meta: KiroCliSessionMeta, entries: KiroCliEntry[], seen for (const item of content) { const rec = asRecord(item) if (rec && rec['kind'] === 'text' && typeof rec['data'] === 'string') { - pendingUserMessage = (rec['data'] as string).slice(0, 500) + pendingUserMessage = flatSlice(rec['data'] as string, 500) inputChars += (rec['data'] as string).length } } @@ -605,7 +610,7 @@ async function parseWorkspaceSession(record: Record, source: Se const text = extractText(msg['content']) if (role === 'user' && text) { inputChars += text.length - pendingUserMessage = text.slice(0, 500) + pendingUserMessage = flatSlice(text, 500) } else if (role === 'assistant' && !execBacked && text && text !== 'On it.') { // An item carrying an executionId is execution-backed: its content is // counted from the execution file, so counting it here would double-count. @@ -662,6 +667,9 @@ async function parseWorkspaceSession(record: Record, source: Se deduplicationKey: dedupKey, userMessage: pendingUserMessage, sessionId, + ...(typeof record['workspaceDirectory'] === 'string' && record['workspaceDirectory'] + ? { projectPath: record['workspaceDirectory'] as string } + : {}), }) return results @@ -774,6 +782,7 @@ async function parseV2Session(source: SessionSource, seenKeys: Set): Pro userMessage: turnUserMessage, sessionId, project: source.project, + ...(meta.workspacePaths?.[0] ? { projectPath: meta.workspacePaths[0] } : {}), }) } } @@ -794,7 +803,7 @@ async function parseV2Session(source: SessionSource, seenKeys: Set): Pro // for the upcoming turn_start. if (inTurn) flushTurn() const text = typeof payload['content'] === 'string' ? payload['content'] as string : extractText(payload['content']) - pendingUserMessage = text.slice(0, 500) + pendingUserMessage = flatSlice(text, 500) pendingUserChars = text.length } else if (type === 'turn_start') { if (inTurn) flushTurn() diff --git a/src/session-cache.ts b/src/session-cache.ts index 2759520f..7be86a33 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -269,7 +269,12 @@ export const PROVIDER_PARSE_VERSIONS: Record = { hermes: 'reasoning-output-accounting-v1-est-cost', 'lingtai-tui': 'token-ledger-registry-activity-v3', 'ibm-bob': 'worktree-project-grouping-v1', - kiro: 'ide-parsing-v1-est-cost', + // project-path-v1: the parser now records the session's full working + // directory as projectPath (CLI meta.cwd, v2 workspacePaths[0], workspace + // sessions' workspaceDirectory), which sync attribution needs to resolve + // the git repo. Cached entries from before the bump lack projectPath and + // would serve attribution-blind sessions forever without a re-parse. + kiro: 'ide-parsing-v1-est-cost-project-path-v1', opencode: 'session-model-v1', quickdesk: 'emf-sqlite-v2-est-cost', kimicode: 'wire-usage-v1-est-cost', diff --git a/tests/flat-slice.test.ts b/tests/flat-slice.test.ts new file mode 100644 index 00000000..78d93bac --- /dev/null +++ b/tests/flat-slice.test.ts @@ -0,0 +1,99 @@ +/** + * Tests for flatSlice — the SlicedString-retention fix. + * + * Background: `String.prototype.slice` returns a V8 SlicedString that + * retains its entire parent string. Storing short slices of large session + * strings (100KB+ agent prompts) in the long-lived session cache pinned + * gigabytes of parent buffers during cold parses, OOMing the default heap + * (issue observed at ~5.5GB peak for 3.2GB of kiro session files; ~300MB + * after flattening). + */ + +import { describe, it, expect } from 'vitest' + +import { flatSlice, flatString } from '../src/content-utils.js' + +describe('flatSlice', () => { + it('returns the prefix for strings over the bound', () => { + const big = 'x'.repeat(10_000) + const out = flatSlice(big, 500) + expect(out.length).toBe(500) + expect(out).toBe(big.slice(0, 500)) + }) + + it('returns the string itself when within the bound', () => { + const small = 'hello world' + expect(flatSlice(small, 500)).toBe(small) + }) + + it('handles multi-byte characters without corruption', () => { + // Emoji + CJK near the boundary — Buffer round-trip must not produce + // invalid UTF-8 replacement chars for chars fully inside the slice. + const s = '🐾'.repeat(300) // each emoji is 2 UTF-16 code units + const out = flatSlice(s, 500) + expect(out).toBe(s.slice(0, 500)) + }) + + it('documents the mid-surrogate-pair cut behavior (U+FFFD)', () => { + // A cut landing between the high and low surrogate of a pair leaves a + // lone surrogate. Plain .slice() preserves it; the Buffer round-trip + // replaces it with U+FFFD. Either way the string is length-bounded and + // the preceding content is intact — this test pins the chosen behavior + // so a future implementation change is a conscious decision. + const s = 'ab' + '🐾'.repeat(300) // odd offset puts every emoji across even boundaries + const out = flatSlice(s, 501) // cuts mid-pair + expect(out.length).toBe(501) + expect(out.slice(0, 500)).toBe(s.slice(0, 500)) // content before the cut intact + expect(out.charCodeAt(500)).toBe(0xfffd) // lone surrogate became U+FFFD + }) + + it('does not retain the parent string (heap growth stays bounded)', () => { + // Property test for the retention fix: keep 1000 short prefixes of + // 1000 distinct 100KB strings. With plain .slice() each prefix pins its + // 100KB parent (~200MB in UTF-16 total). With flatSlice, retained data + // is ~1000 × 500 chars ≈ 1MB. Assert heap growth is far below the + // retention scenario. Threshold is generous (50MB) to be CI-safe while + // still failing decisively if retention returns (>190MB). When the test + // runner exposes gc (vitest under --expose-gc), force a collection so + // transient parent garbage doesn't inflate the measurement. + const before = process.memoryUsage().heapUsed + const kept: string[] = [] + for (let i = 0; i < 1000; i++) { + // Distinct content so V8 cannot intern/share the parents. + const parent = (i % 10).toString().repeat(100_000) + kept.push(flatSlice(parent + i, 500)) + } + if (typeof global.gc === 'function') global.gc() + const after = process.memoryUsage().heapUsed + const growthMB = (after - before) / 1048576 + expect(kept.length).toBe(1000) + expect(growthMB).toBeLessThan(50) + }) +}) + +describe('flatString', () => { + it('returns an equal string for any input', () => { + expect(flatString('')).toBe('') + expect(flatString('hello')).toBe('hello') + expect(flatString('🐾 multi-byte ✓')).toBe('🐾 multi-byte ✓') + }) + + it('does not retain the parent of a regex match group', () => { + // match[1] is a SlicedString retaining the entire subject. flatString + // must break that link: keep 1000 short match groups of distinct 100KB + // subjects and assert bounded heap growth (same thresholds as the + // flatSlice retention test). + const before = process.memoryUsage().heapUsed + const kept: string[] = [] + for (let i = 0; i < 1000; i++) { + const subject = `tool_${i}` + (i % 10).toString().repeat(100_000) + const m = /([^<]+)<\/name>/.exec(subject) + kept.push(flatString(m![1]!)) + } + if (typeof global.gc === 'function') global.gc() + const after = process.memoryUsage().heapUsed + const growthMB = (after - before) / 1048576 + expect(kept.length).toBe(1000) + expect(growthMB).toBeLessThan(50) + }) +}) diff --git a/tests/kiro-projectpath.test.ts b/tests/kiro-projectpath.test.ts new file mode 100644 index 00000000..8d14e8da --- /dev/null +++ b/tests/kiro-projectpath.test.ts @@ -0,0 +1,205 @@ +/** + * Tests for kiro projectPath emission (sync attribution support). + * + * The kiro provider historically reduced the session's working directory to + * `basename(cwd)` for display and discarded the full path. Sync attribution + * (`codeburn sync push --attribution`) needs the full path on + * `ParsedProviderCall.projectPath` to resolve the git repo — without it, + * every kiro session is attribution-blind. + * + * Also covers the cache side: projectPath is persisted via CachedCall, so + * entries cached BEFORE the parser learned to emit it must re-parse. That is + * driven by the PROVIDER_PARSE_VERSIONS.kiro bump (project-path-v1); a cache + * seeded at the pre-bump fingerprint must be discarded. + */ + +import { mkdir, writeFile, rm } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { join } from 'node:path' + +import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest' + +import { clearSessionCache, parseAllSessions } from '../src/parser.js' +import { + CACHE_VERSION, + computeEnvFingerprint, + fingerprintFile, + sessionCachePath, + type SessionCache, +} from '../src/session-cache.js' + +// The kiro provider reads homedir()/env at call time in discovery; HOME must +// point at the test root before ../src/parser.js is evaluated (see the +// equivalent note in kiro-cache-invalidation.test.ts). +const testRoot = vi.hoisted(() => { + const root = `${process.env['TMPDIR'] || '/tmp'}/kiro-projpath-${process.pid}-${Date.now()}` + process.env['HOME'] = `${root}/home` + process.env['USERPROFILE'] = `${root}/home` + return root +}) + +const HOME = join(testRoot, 'home') +const CACHE_DIR = join(testRoot, 'cache') +const KIRO_SESSIONS = join(HOME, '.kiro', 'sessions') +const CLI_DIR = join(KIRO_SESSIONS, 'cli') + +const CLI_CWD = '/local/home/testuser/workplace/my-project' +const V2_WORKSPACE = '/local/home/testuser/workplace/ide-project' + +beforeEach(() => { + process.env['HOME'] = HOME + process.env['USERPROFILE'] = HOME + process.env['CODEBURN_CACHE_DIR'] = CACHE_DIR + delete process.env['KIRO_HOME'] + clearSessionCache() +}) + +afterAll(async () => { + await rm(testRoot, { recursive: true, force: true }) +}) + +/** Write a minimal kiro CLI session: .jsonl entries + companion .json meta. */ +async function seedCliSession(id: string, cwd: string): Promise { + await mkdir(CLI_DIR, { recursive: true }) + const jsonlPath = join(CLI_DIR, `${id}.jsonl`) + const entries = [ + { kind: 'Prompt', data: { content: [{ kind: 'text', data: 'add a feature' }] } }, + { kind: 'AssistantMessage', data: { content: [{ kind: 'text', data: 'Done — added the feature and tests.' }] } }, + ] + await writeFile(jsonlPath, entries.map(e => JSON.stringify(e)).join('\n')) + await writeFile(join(CLI_DIR, `${id}.json`), JSON.stringify({ + session_id: id, + cwd, + created_at: '2026-08-01T10:00:00Z', + updated_at: '2026-08-01T10:05:00Z', + session_state: { + rts_model_state: { model_info: { model_id: 'auto' } }, + conversation_metadata: { + user_turn_metadatas: [ + { end_timestamp: '2026-08-01T10:05:00Z', metering_usage: [] }, + ], + }, + }, + })) + return jsonlPath +} + +/** Write a minimal v2 IDE session: sessions//sess_/{session.json,messages.jsonl}. */ +async function seedV2Session(id: string, workspacePath: string): Promise { + const sessDir = join(KIRO_SESSIONS, 'f'.repeat(32), `sess_${id}`) + await mkdir(sessDir, { recursive: true }) + await writeFile(join(sessDir, 'session.json'), JSON.stringify({ + id, + modelId: 'auto', + workspacePaths: [workspacePath], + createdAt: '2026-08-01T11:00:00Z', + })) + const events = [ + { timestamp: '2026-08-01T11:00:00Z', payload: { type: 'user', content: 'fix the bug' } }, + { timestamp: '2026-08-01T11:00:01Z', payload: { type: 'turn_start', executionId: 'x1' } }, + { timestamp: '2026-08-01T11:00:05Z', payload: { type: 'assistant', content: 'Fixed the bug in handler.ts by checking null first.' } }, + { timestamp: '2026-08-01T11:00:06Z', payload: { type: 'turn_end', executionId: 'x1' } }, + ] + await writeFile(join(sessDir, 'messages.jsonl'), events.map(e => JSON.stringify(e)).join('\n')) +} + +function kiroAgentDir(): string { + if (process.platform === 'darwin') { + return join(HOME, 'Library', 'Application Support', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent') + } + if (process.platform === 'win32') { + return join(HOME, 'AppData', 'Roaming', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent') + } + return join(HOME, '.config', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent') +} + +/** Write a minimal IDE workspace-session: + * /workspace-sessions//.json */ +async function seedWorkspaceSession(id: string, workspaceDirectory: string): Promise { + const encoded = Buffer.from(workspaceDirectory, 'utf-8').toString('base64').replace(/=/g, '_') + const dir = join(kiroAgentDir(), 'workspace-sessions', encoded) + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, `${id}.json`), JSON.stringify({ + sessionId: id, + selectedModel: 'auto', + workspaceDirectory, + history: [ + { message: { role: 'user', content: 'refactor the config loader' } }, + { message: { role: 'assistant', content: 'Refactored the loader into three small functions with tests.' } }, + ], + })) +} + +async function kiroCalls() { + const projects = await parseAllSessions(undefined, 'kiro') + return projects.flatMap(p => p.sessions.map(s => ({ project: p.project, projectPath: p.projectPath, session: s }))) +} + +describe('kiro projectPath emission', () => { + it('CLI session: projectPath is the full meta.cwd, project the basename', async () => { + await seedCliSession('cli-001', CLI_CWD) + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'my-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(CLI_CWD) + }) + + it('v2 IDE session: projectPath is workspacePaths[0]', async () => { + await seedV2Session('v2-001', V2_WORKSPACE) + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'ide-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(V2_WORKSPACE) + }) + + it('workspace session: projectPath is workspaceDirectory', async () => { + const WS_DIR = '/local/home/testuser/workplace/ws-project' + await seedWorkspaceSession('ws-001', WS_DIR) + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'ws-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(WS_DIR) + }) +}) + +describe('kiro projectPath cache invalidation (project-path-v1 bump)', () => { + // The fingerprint a cache written by the PREVIOUS release carries: same env + // vars, but the parser version before the project-path-v1 bump. + function preBumpFingerprint(): string { + const parts = [`KIRO_HOME=${process.env['KIRO_HOME'] ?? ''}`, 'parser=ide-parsing-v1-est-cost'] + return createHash('sha256').update(parts.join('\0')).digest('hex').slice(0, 16) + } + + it('the bump changed the env fingerprint', () => { + expect(computeEnvFingerprint('kiro')).not.toBe(preBumpFingerprint()) + }) + + it('a pre-bump cache entry (no projectPath) is re-parsed and gains projectPath', async () => { + const jsonlPath = await seedCliSession('cli-002', CLI_CWD) + + // Seed a cache exactly as the pre-bump release would have left it: + // correct file fingerprint, pre-bump env fingerprint, turns WITHOUT + // projectPath on the cached calls. + const fp = await fingerprintFile(jsonlPath) + if (!fp) throw new Error('failed to fingerprint seeded session file') + const cache: SessionCache = { + version: CACHE_VERSION, + providers: { + kiro: { + envFingerprint: preBumpFingerprint(), + files: { + [jsonlPath]: { fingerprint: fp, mcpInventory: [], turns: [] }, + }, + }, + }, + } + await mkdir(CACHE_DIR, { recursive: true }) + await writeFile(sessionCachePath(), JSON.stringify(cache)) + clearSessionCache() + + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'my-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(CLI_CWD) + }) +})