From d47a1711b1eff2a39781a18ad821c9bc10c226de Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Wed, 12 Aug 2026 15:54:10 +0000 Subject: [PATCH 1/2] =?UTF-8?q?fix(parser):=20OOM=20on=20cold=20parse=20?= =?UTF-8?q?=E2=80=94=20V8=20SlicedString=20retention=20in=20session=20cach?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String.prototype.slice returns a V8 SlicedString: a view that retains a reference to its ENTIRE parent string. The parsers store short previews of message text (userMessage.slice(0, 500/2000)) in the long-lived session cache. Session files routinely carry 100KB+ strings (agent- injected system prompts, tool results), so every cached preview pinned its full parent buffer for the life of the process. Measured on 3.2GB of kiro CLI session files (6,659 files, largest 40MB): cold parse, default heap, before: 4.33GB peak -> OOM crash cold parse, 8GB heap, before: 5.67GB peak (kiro provider alone) after kiro flatSlice: 0.91GB peak after parser.ts cache sites too: 0.64GB peak original failing command (cold, default heap, all providers): 0.89GB peak -> completes Warm runs were always fine (~0.29GB) because the cache's JSON round-trip flattens the strings on load — which made this bug appear intermittent: it only fired on a cold or invalidated cache. Fix: flatSlice() in content-utils.ts forces a flat copy via Buffer round-trip. Applied at the six kiro userMessage capture sites and the three shared cache-building sites in parser.ts (protects all providers). Regression test asserts the no-retention property via bounded heap growth over 1000 large-parent slices. AI-Origin: human --- src/content-utils.ts | 31 +++++++++++++ src/parser.ts | 12 ++--- src/providers/kiro.ts | 18 +++++--- tests/flat-slice.test.ts | 99 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 tests/flat-slice.test.ts diff --git a/src/content-utils.ts b/src/content-utils.ts index 5ca31ae9..f3717194 100644 --- a/src/content-utils.ts +++ b/src/content-utils.ts @@ -24,3 +24,34 @@ export function normalizeContentBlocks)['aiTitle'] - if (typeof t === 'string' && t.trim()) meta.title = t.trim().slice(0, 200) + if (typeof t === 'string' && t.trim()) meta.title = flatString(t.trim().slice(0, 200)) } else if (entry.type === 'pr-link') { const url = (entry as Record)['prUrl'] if (typeof url === 'string' && url && !meta.prLinks.includes(url)) meta.prLinks.push(url) @@ -1900,7 +1900,7 @@ export async function readAgentType(filePath: string): Promise')) continue inputChars += msg.content.length - pendingUserMessage = msg.content.slice(0, 500) + pendingUserMessage = flatSlice(msg.content, 500) } if (msg.role === 'bot') { const msgTools = extractToolNames(msg.content) @@ -296,7 +300,7 @@ function parseModernExecution(data: KiroModernExecution, sourcePath: string, see if (directInput) { inputChars += directInput.length - pendingUserMessage = directInput.slice(0, 500) + pendingUserMessage = flatSlice(directInput, 500) } if (directOutput) { @@ -328,7 +332,7 @@ function parseModernExecution(data: KiroModernExecution, sourcePath: string, see if (role === 'human' || role === 'user') { if (!text) continue inputChars += text.length - pendingUserMessage = text.slice(0, 500) + pendingUserMessage = flatSlice(text, 500) } else if (role === 'bot' || role === 'assistant' || role === 'ai' || role === 'model') { if (text) outputChars += text.length if (text || tools.length > 0) hasOutputActivity = true @@ -526,7 +530,7 @@ function parseCliSession(meta: KiroCliSessionMeta, entries: KiroCliEntry[], seen for (const item of content) { const rec = asRecord(item) if (rec && rec['kind'] === 'text' && typeof rec['data'] === 'string') { - pendingUserMessage = (rec['data'] as string).slice(0, 500) + pendingUserMessage = flatSlice(rec['data'] as string, 500) inputChars += (rec['data'] as string).length } } @@ -605,7 +609,7 @@ async function parseWorkspaceSession(record: Record, source: Se const text = extractText(msg['content']) if (role === 'user' && text) { inputChars += text.length - pendingUserMessage = text.slice(0, 500) + pendingUserMessage = flatSlice(text, 500) } else if (role === 'assistant' && !execBacked && text && text !== 'On it.') { // An item carrying an executionId is execution-backed: its content is // counted from the execution file, so counting it here would double-count. @@ -794,7 +798,7 @@ async function parseV2Session(source: SessionSource, seenKeys: Set): Pro // for the upcoming turn_start. if (inTurn) flushTurn() const text = typeof payload['content'] === 'string' ? payload['content'] as string : extractText(payload['content']) - pendingUserMessage = text.slice(0, 500) + pendingUserMessage = flatSlice(text, 500) pendingUserChars = text.length } else if (type === 'turn_start') { if (inTurn) flushTurn() diff --git a/tests/flat-slice.test.ts b/tests/flat-slice.test.ts new file mode 100644 index 00000000..78d93bac --- /dev/null +++ b/tests/flat-slice.test.ts @@ -0,0 +1,99 @@ +/** + * Tests for flatSlice — the SlicedString-retention fix. + * + * Background: `String.prototype.slice` returns a V8 SlicedString that + * retains its entire parent string. Storing short slices of large session + * strings (100KB+ agent prompts) in the long-lived session cache pinned + * gigabytes of parent buffers during cold parses, OOMing the default heap + * (issue observed at ~5.5GB peak for 3.2GB of kiro session files; ~300MB + * after flattening). + */ + +import { describe, it, expect } from 'vitest' + +import { flatSlice, flatString } from '../src/content-utils.js' + +describe('flatSlice', () => { + it('returns the prefix for strings over the bound', () => { + const big = 'x'.repeat(10_000) + const out = flatSlice(big, 500) + expect(out.length).toBe(500) + expect(out).toBe(big.slice(0, 500)) + }) + + it('returns the string itself when within the bound', () => { + const small = 'hello world' + expect(flatSlice(small, 500)).toBe(small) + }) + + it('handles multi-byte characters without corruption', () => { + // Emoji + CJK near the boundary — Buffer round-trip must not produce + // invalid UTF-8 replacement chars for chars fully inside the slice. + const s = '🐾'.repeat(300) // each emoji is 2 UTF-16 code units + const out = flatSlice(s, 500) + expect(out).toBe(s.slice(0, 500)) + }) + + it('documents the mid-surrogate-pair cut behavior (U+FFFD)', () => { + // A cut landing between the high and low surrogate of a pair leaves a + // lone surrogate. Plain .slice() preserves it; the Buffer round-trip + // replaces it with U+FFFD. Either way the string is length-bounded and + // the preceding content is intact — this test pins the chosen behavior + // so a future implementation change is a conscious decision. + const s = 'ab' + '🐾'.repeat(300) // odd offset puts every emoji across even boundaries + const out = flatSlice(s, 501) // cuts mid-pair + expect(out.length).toBe(501) + expect(out.slice(0, 500)).toBe(s.slice(0, 500)) // content before the cut intact + expect(out.charCodeAt(500)).toBe(0xfffd) // lone surrogate became U+FFFD + }) + + it('does not retain the parent string (heap growth stays bounded)', () => { + // Property test for the retention fix: keep 1000 short prefixes of + // 1000 distinct 100KB strings. With plain .slice() each prefix pins its + // 100KB parent (~200MB in UTF-16 total). With flatSlice, retained data + // is ~1000 × 500 chars ≈ 1MB. Assert heap growth is far below the + // retention scenario. Threshold is generous (50MB) to be CI-safe while + // still failing decisively if retention returns (>190MB). When the test + // runner exposes gc (vitest under --expose-gc), force a collection so + // transient parent garbage doesn't inflate the measurement. + const before = process.memoryUsage().heapUsed + const kept: string[] = [] + for (let i = 0; i < 1000; i++) { + // Distinct content so V8 cannot intern/share the parents. + const parent = (i % 10).toString().repeat(100_000) + kept.push(flatSlice(parent + i, 500)) + } + if (typeof global.gc === 'function') global.gc() + const after = process.memoryUsage().heapUsed + const growthMB = (after - before) / 1048576 + expect(kept.length).toBe(1000) + expect(growthMB).toBeLessThan(50) + }) +}) + +describe('flatString', () => { + it('returns an equal string for any input', () => { + expect(flatString('')).toBe('') + expect(flatString('hello')).toBe('hello') + expect(flatString('🐾 multi-byte ✓')).toBe('🐾 multi-byte ✓') + }) + + it('does not retain the parent of a regex match group', () => { + // match[1] is a SlicedString retaining the entire subject. flatString + // must break that link: keep 1000 short match groups of distinct 100KB + // subjects and assert bounded heap growth (same thresholds as the + // flatSlice retention test). + const before = process.memoryUsage().heapUsed + const kept: string[] = [] + for (let i = 0; i < 1000; i++) { + const subject = `tool_${i}` + (i % 10).toString().repeat(100_000) + const m = /([^<]+)<\/name>/.exec(subject) + kept.push(flatString(m![1]!)) + } + if (typeof global.gc === 'function') global.gc() + const after = process.memoryUsage().heapUsed + const growthMB = (after - before) / 1048576 + expect(kept.length).toBe(1000) + expect(growthMB).toBeLessThan(50) + }) +}) From db35fdd0797474f900f3f0c3b34a0a38adb6cc36 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Wed, 12 Aug 2026 15:10:31 +0000 Subject: [PATCH 2/2] fix(kiro): set projectPath from session cwd for attribution support The kiro provider was reading the full working directory from session metadata (meta.cwd for CLI sessions, workspacePaths[0] for v2 IDE sessions, workspaceDirectory for workspace sessions) but discarding it via basename(), keeping only the leaf name for display. This meant computeAttributionRecords could never resolve kiro sessions to a git repo, so `codeburn sync push --attribution` produced 0 facts for all kiro-originated sessions. Now passes the full path as projectPath on emitted ParsedProviderCalls, which buildRepoGroups uses to resolve git identity and correlate commits with sessions via timestamp windows. The path stays local: only the normalized origin remote egresses in attribution spans. Behavior changes beyond attribution: - kiro calls now flow through canonicalizeProviderCallProject, so kiro sessions in LINKED GIT WORKTREES canonicalize to the main repository: their report project name changes from the worktree dir name to the main repo name (consistent with claude/codex behavior). - workingDirectory is now populated on kiro calls. - PROVIDER_PARSE_VERSIONS.kiro bumped (project-path-v1): cached entries predate projectPath and are served without re-invoking the parser, so without the bump this fix silently no-ops for every warm cache. The bump forces a one-time cold kiro re-parse on upgrade. ORDERING: this commit must land WITH (or after) the preceding SlicedString OOM fix. The forced cold re-parse it triggers is exactly the workload that OOM'd before that fix on multi-GB kiro stores. Perf: per-call canonicalization added a measured +5% to cold parse (.git-marker lstat walk per call). resolveCanonicalProjectPath is now memoized on cwd (cleared with the session cache), removing the redundant walks for all providers. Tests: projectPath emission fixtures for all three session formats (CLI, v2 IDE, workspace-session), fingerprint-change assertion, and a regression test seeding a pre-bump cache entry and proving the re-parse recovers projectPath. AI-Origin: human --- src/parser.ts | 18 +++ src/providers/kiro.ts | 5 + src/session-cache.ts | 7 +- tests/kiro-projectpath.test.ts | 205 +++++++++++++++++++++++++++++++++ 4 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 tests/kiro-projectpath.test.ts diff --git a/src/parser.ts b/src/parser.ts index 9fd7f034..2fc11eb4 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -91,7 +91,24 @@ function isCoworkSession(cwd: string, filePath: string): boolean { }) } +// Memoizes resolveCanonicalProjectPath: every ParsedProviderCall with a +// projectPath pays the .git-marker directory walk (one lstat per ancestor +// level), and a session's calls all share one cwd — without this cache a +// cold parse re-walks the same few directories thousands of times +// (measured ~+5% cold-parse time for a large kiro store). Filesystem facts +// can go stale in a long-lived process (a dir converted to a worktree +// mid-run), so the cache is cleared with the session cache. +const canonicalPathCache = new Map() + async function resolveCanonicalProjectPath(cwd: string): Promise<{ path: string; isWorktree: boolean }> { + const cached = canonicalPathCache.get(cwd) + if (cached) return cached + const result = await resolveCanonicalProjectPathUncached(cwd) + canonicalPathCache.set(cwd, result) + return result +} + +async function resolveCanonicalProjectPathUncached(cwd: string): Promise<{ path: string; isWorktree: boolean }> { const trimmed = cwd.trim() if (!trimmed) return { path: cwd, isWorktree: false } @@ -3253,6 +3270,7 @@ function cacheKey(dateRange?: DateRange, providerFilter?: string): string { export function clearSessionCache(): void { sessionCache.clear() + canonicalPathCache.clear() } function cachePut(key: string, data: ProjectSummary[]) { diff --git a/src/providers/kiro.ts b/src/providers/kiro.ts index 870b72ca..4e0308d9 100644 --- a/src/providers/kiro.ts +++ b/src/providers/kiro.ts @@ -510,6 +510,7 @@ function parseCliSession(meta: KiroCliSessionMeta, entries: KiroCliEntry[], seen userMessage: pendingUserMessage, sessionId, project, + ...(meta.cwd ? { projectPath: meta.cwd } : {}), }) turnIndex++ } @@ -666,6 +667,9 @@ async function parseWorkspaceSession(record: Record, source: Se deduplicationKey: dedupKey, userMessage: pendingUserMessage, sessionId, + ...(typeof record['workspaceDirectory'] === 'string' && record['workspaceDirectory'] + ? { projectPath: record['workspaceDirectory'] as string } + : {}), }) return results @@ -778,6 +782,7 @@ async function parseV2Session(source: SessionSource, seenKeys: Set): Pro userMessage: turnUserMessage, sessionId, project: source.project, + ...(meta.workspacePaths?.[0] ? { projectPath: meta.workspacePaths[0] } : {}), }) } } diff --git a/src/session-cache.ts b/src/session-cache.ts index 2759520f..7be86a33 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -269,7 +269,12 @@ export const PROVIDER_PARSE_VERSIONS: Record = { hermes: 'reasoning-output-accounting-v1-est-cost', 'lingtai-tui': 'token-ledger-registry-activity-v3', 'ibm-bob': 'worktree-project-grouping-v1', - kiro: 'ide-parsing-v1-est-cost', + // project-path-v1: the parser now records the session's full working + // directory as projectPath (CLI meta.cwd, v2 workspacePaths[0], workspace + // sessions' workspaceDirectory), which sync attribution needs to resolve + // the git repo. Cached entries from before the bump lack projectPath and + // would serve attribution-blind sessions forever without a re-parse. + kiro: 'ide-parsing-v1-est-cost-project-path-v1', opencode: 'session-model-v1', quickdesk: 'emf-sqlite-v2-est-cost', kimicode: 'wire-usage-v1-est-cost', diff --git a/tests/kiro-projectpath.test.ts b/tests/kiro-projectpath.test.ts new file mode 100644 index 00000000..8d14e8da --- /dev/null +++ b/tests/kiro-projectpath.test.ts @@ -0,0 +1,205 @@ +/** + * Tests for kiro projectPath emission (sync attribution support). + * + * The kiro provider historically reduced the session's working directory to + * `basename(cwd)` for display and discarded the full path. Sync attribution + * (`codeburn sync push --attribution`) needs the full path on + * `ParsedProviderCall.projectPath` to resolve the git repo — without it, + * every kiro session is attribution-blind. + * + * Also covers the cache side: projectPath is persisted via CachedCall, so + * entries cached BEFORE the parser learned to emit it must re-parse. That is + * driven by the PROVIDER_PARSE_VERSIONS.kiro bump (project-path-v1); a cache + * seeded at the pre-bump fingerprint must be discarded. + */ + +import { mkdir, writeFile, rm } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { join } from 'node:path' + +import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest' + +import { clearSessionCache, parseAllSessions } from '../src/parser.js' +import { + CACHE_VERSION, + computeEnvFingerprint, + fingerprintFile, + sessionCachePath, + type SessionCache, +} from '../src/session-cache.js' + +// The kiro provider reads homedir()/env at call time in discovery; HOME must +// point at the test root before ../src/parser.js is evaluated (see the +// equivalent note in kiro-cache-invalidation.test.ts). +const testRoot = vi.hoisted(() => { + const root = `${process.env['TMPDIR'] || '/tmp'}/kiro-projpath-${process.pid}-${Date.now()}` + process.env['HOME'] = `${root}/home` + process.env['USERPROFILE'] = `${root}/home` + return root +}) + +const HOME = join(testRoot, 'home') +const CACHE_DIR = join(testRoot, 'cache') +const KIRO_SESSIONS = join(HOME, '.kiro', 'sessions') +const CLI_DIR = join(KIRO_SESSIONS, 'cli') + +const CLI_CWD = '/local/home/testuser/workplace/my-project' +const V2_WORKSPACE = '/local/home/testuser/workplace/ide-project' + +beforeEach(() => { + process.env['HOME'] = HOME + process.env['USERPROFILE'] = HOME + process.env['CODEBURN_CACHE_DIR'] = CACHE_DIR + delete process.env['KIRO_HOME'] + clearSessionCache() +}) + +afterAll(async () => { + await rm(testRoot, { recursive: true, force: true }) +}) + +/** Write a minimal kiro CLI session: .jsonl entries + companion .json meta. */ +async function seedCliSession(id: string, cwd: string): Promise { + await mkdir(CLI_DIR, { recursive: true }) + const jsonlPath = join(CLI_DIR, `${id}.jsonl`) + const entries = [ + { kind: 'Prompt', data: { content: [{ kind: 'text', data: 'add a feature' }] } }, + { kind: 'AssistantMessage', data: { content: [{ kind: 'text', data: 'Done — added the feature and tests.' }] } }, + ] + await writeFile(jsonlPath, entries.map(e => JSON.stringify(e)).join('\n')) + await writeFile(join(CLI_DIR, `${id}.json`), JSON.stringify({ + session_id: id, + cwd, + created_at: '2026-08-01T10:00:00Z', + updated_at: '2026-08-01T10:05:00Z', + session_state: { + rts_model_state: { model_info: { model_id: 'auto' } }, + conversation_metadata: { + user_turn_metadatas: [ + { end_timestamp: '2026-08-01T10:05:00Z', metering_usage: [] }, + ], + }, + }, + })) + return jsonlPath +} + +/** Write a minimal v2 IDE session: sessions//sess_/{session.json,messages.jsonl}. */ +async function seedV2Session(id: string, workspacePath: string): Promise { + const sessDir = join(KIRO_SESSIONS, 'f'.repeat(32), `sess_${id}`) + await mkdir(sessDir, { recursive: true }) + await writeFile(join(sessDir, 'session.json'), JSON.stringify({ + id, + modelId: 'auto', + workspacePaths: [workspacePath], + createdAt: '2026-08-01T11:00:00Z', + })) + const events = [ + { timestamp: '2026-08-01T11:00:00Z', payload: { type: 'user', content: 'fix the bug' } }, + { timestamp: '2026-08-01T11:00:01Z', payload: { type: 'turn_start', executionId: 'x1' } }, + { timestamp: '2026-08-01T11:00:05Z', payload: { type: 'assistant', content: 'Fixed the bug in handler.ts by checking null first.' } }, + { timestamp: '2026-08-01T11:00:06Z', payload: { type: 'turn_end', executionId: 'x1' } }, + ] + await writeFile(join(sessDir, 'messages.jsonl'), events.map(e => JSON.stringify(e)).join('\n')) +} + +function kiroAgentDir(): string { + if (process.platform === 'darwin') { + return join(HOME, 'Library', 'Application Support', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent') + } + if (process.platform === 'win32') { + return join(HOME, 'AppData', 'Roaming', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent') + } + return join(HOME, '.config', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent') +} + +/** Write a minimal IDE workspace-session: + * /workspace-sessions//.json */ +async function seedWorkspaceSession(id: string, workspaceDirectory: string): Promise { + const encoded = Buffer.from(workspaceDirectory, 'utf-8').toString('base64').replace(/=/g, '_') + const dir = join(kiroAgentDir(), 'workspace-sessions', encoded) + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, `${id}.json`), JSON.stringify({ + sessionId: id, + selectedModel: 'auto', + workspaceDirectory, + history: [ + { message: { role: 'user', content: 'refactor the config loader' } }, + { message: { role: 'assistant', content: 'Refactored the loader into three small functions with tests.' } }, + ], + })) +} + +async function kiroCalls() { + const projects = await parseAllSessions(undefined, 'kiro') + return projects.flatMap(p => p.sessions.map(s => ({ project: p.project, projectPath: p.projectPath, session: s }))) +} + +describe('kiro projectPath emission', () => { + it('CLI session: projectPath is the full meta.cwd, project the basename', async () => { + await seedCliSession('cli-001', CLI_CWD) + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'my-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(CLI_CWD) + }) + + it('v2 IDE session: projectPath is workspacePaths[0]', async () => { + await seedV2Session('v2-001', V2_WORKSPACE) + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'ide-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(V2_WORKSPACE) + }) + + it('workspace session: projectPath is workspaceDirectory', async () => { + const WS_DIR = '/local/home/testuser/workplace/ws-project' + await seedWorkspaceSession('ws-001', WS_DIR) + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'ws-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(WS_DIR) + }) +}) + +describe('kiro projectPath cache invalidation (project-path-v1 bump)', () => { + // The fingerprint a cache written by the PREVIOUS release carries: same env + // vars, but the parser version before the project-path-v1 bump. + function preBumpFingerprint(): string { + const parts = [`KIRO_HOME=${process.env['KIRO_HOME'] ?? ''}`, 'parser=ide-parsing-v1-est-cost'] + return createHash('sha256').update(parts.join('\0')).digest('hex').slice(0, 16) + } + + it('the bump changed the env fingerprint', () => { + expect(computeEnvFingerprint('kiro')).not.toBe(preBumpFingerprint()) + }) + + it('a pre-bump cache entry (no projectPath) is re-parsed and gains projectPath', async () => { + const jsonlPath = await seedCliSession('cli-002', CLI_CWD) + + // Seed a cache exactly as the pre-bump release would have left it: + // correct file fingerprint, pre-bump env fingerprint, turns WITHOUT + // projectPath on the cached calls. + const fp = await fingerprintFile(jsonlPath) + if (!fp) throw new Error('failed to fingerprint seeded session file') + const cache: SessionCache = { + version: CACHE_VERSION, + providers: { + kiro: { + envFingerprint: preBumpFingerprint(), + files: { + [jsonlPath]: { fingerprint: fp, mcpInventory: [], turns: [] }, + }, + }, + }, + } + await mkdir(CACHE_DIR, { recursive: true }) + await writeFile(sessionCachePath(), JSON.stringify(cache)) + clearSessionCache() + + const rows = await kiroCalls() + const row = rows.find(r => r.project === 'my-project') + expect(row).toBeDefined() + expect(row!.projectPath).toBe(CLI_CWD) + }) +})