From 516faa82c81978a179ae307dee52b0e7cc7dd0a0 Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Wed, 12 Aug 2026 12:46:26 -0700 Subject: [PATCH] calibrate: opt-in plan-window utilization sampler Records Anthropic's live plan-window utilization (five_hour / seven_day / model-scoped weekly, the same oauth/usage endpoint the menubar polls) into a local JSONL so the deltas can be regressed against the local token record to recover per-model plan-burn weights. - codeburn calibrate --enable/--disable/--sample, status always printed - serve rides a 60s tick; real spacing enforced by a 5-min mtime throttle - token from ~/.claude/.credentials.json, else the macOS keychain item - off by default; samples never leave the machine --- src/config.ts | 6 ++ src/main.ts | 54 ++++++++++ src/usage-sampler.ts | 191 ++++++++++++++++++++++++++++++++++++ tests/usage-sampler.test.ts | 86 ++++++++++++++++ 4 files changed, 337 insertions(+) create mode 100644 src/usage-sampler.ts create mode 100644 tests/usage-sampler.test.ts diff --git a/src/config.ts b/src/config.ts index b042749e..16f621fb 100644 --- a/src/config.ts +++ b/src/config.ts @@ -61,6 +61,12 @@ export type CodeburnConfig = { // Matched against the canonical project path: prefix on a path-segment // boundary, case-insensitive, trailing-slash and backslash tolerant. proxyPaths?: string[] + // Opt-in plan-limit calibration: when enabled, the resident serve process + // periodically records Anthropic's live plan-window utilization to a local + // JSONL (usage-sampler.ts) so plan-burn weights can be fitted later. + calibration?: { + enabled?: boolean + } } function getConfigDir(): string { diff --git a/src/main.ts b/src/main.ts index d201920b..c9a195fb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1374,6 +1374,51 @@ program console.log(` Config saved to ${getConfigFilePath()}\n`) }) +program + .command('calibrate') + .description('Record live plan-window utilization samples for plan-burn calibration (local only)') + .option('--enable', 'Turn sampling on (the resident serve process samples every few minutes)') + .option('--disable', 'Turn sampling off (recorded samples are kept)') + .option('--sample', 'Take one sample right now') + .action(async (opts: { enable?: boolean; disable?: boolean; sample?: boolean }) => { + const { readSamplesInfo, sampleUsageNow, usageSamplesPath } = await import('./usage-sampler.js') + + if (opts.enable || opts.disable) { + const config = await readConfig() + config.calibration = { enabled: Boolean(opts.enable) } + await saveConfig(config) + console.log(`\n Calibration sampling ${opts.enable ? 'enabled' : 'disabled'}.`) + } + + if (opts.enable || opts.sample) { + const outcome = await sampleUsageNow({ force: true }) + if (outcome.ok) { + const parts = [ + outcome.sample.fiveHour ? `5h ${outcome.sample.fiveHour.pct}%` : undefined, + outcome.sample.sevenDay ? `weekly ${outcome.sample.sevenDay.pct}%` : undefined, + ...(outcome.sample.scoped ?? []).map(w => `${w.label} ${w.pct}%`), + ].filter(Boolean) + console.log(` Sampled: ${parts.join(' · ')}`) + } else { + const why: Record = { + 'no-token': 'no Claude Code OAuth token found (run claude once to sign in)', + 'http-error': 'the usage endpoint refused the request', + 'malformed': 'the usage endpoint returned an unexpected shape', + 'network': 'the usage endpoint was unreachable', + 'throttled': 'sampled too recently', + } + console.log(` Sample failed: ${why[outcome.reason] ?? outcome.reason}.`) + if (opts.enable) console.log(' Sampling stays enabled; the serve process retries on its own.') + } + } + + const config = await readConfig() + const info = await readSamplesInfo() + console.log(`\n Status: ${config.calibration?.enabled ? 'enabled' : 'disabled'}`) + console.log(` Samples: ${info.count}${info.firstTs ? ` (${info.firstTs} → ${info.lastTs})` : ''}`) + console.log(` File: ${usageSamplesPath()}\n`) + }) + program .command('model-alias [from] [to]') .description('Map a provider model name to a canonical one for pricing (e.g. codeburn model-alias my-model claude-opus-4-6)') @@ -2360,6 +2405,15 @@ return program if (process.argv[2] === 'serve') { const { runStdioServe } = await import('./serve.js') + // Opt-in calibration rides on the resident process: a low-cadence tick whose + // real spacing is enforced by the sampler's own mtime throttle. unref'd so it + // never keeps serve alive, and every failure mode is a quiet typed outcome. + const calibrationOn = await readConfig().then(c => c.calibration?.enabled === true).catch(() => false) + if (calibrationOn) { + const { sampleUsageNow } = await import('./usage-sampler.js') + void sampleUsageNow().catch(() => {}) + setInterval(() => { void sampleUsageNow().catch(() => {}) }, 60_000).unref() + } await runStdioServe(buildProgram) } else { buildProgram().parse() diff --git a/src/usage-sampler.ts b/src/usage-sampler.ts new file mode 100644 index 00000000..28a00885 --- /dev/null +++ b/src/usage-sampler.ts @@ -0,0 +1,191 @@ +// Plan-limit calibration sampler (opt-in via `codeburn calibrate --enable`). +// Periodically records Anthropic's live plan-window utilization (the same +// oauth/usage endpoint the menubar polls) into a local JSONL so the deltas can +// later be regressed against the token record to recover per-model plan-burn +// weights. Local only: samples never leave the machine. +import { execFile } from 'node:child_process' +import { appendFile, mkdir, readFile, stat } from 'node:fs/promises' +import { homedir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) + +const USAGE_URL = 'https://api.anthropic.com/api/oauth/usage' +const BETA_HEADER = 'oauth-2025-04-20' +const USER_AGENT = 'claude-code/2.1.0' +const KEYCHAIN_SERVICE = 'Claude Code-credentials' + +/** Minimum spacing between samples; the serve tick calls in more often and + * relies on this throttle, keyed off the samples file's mtime. */ +export const SAMPLE_MIN_INTERVAL_MS = 5 * 60 * 1000 + +type WindowSample = { pct: number; resetsAt?: string } + +export type UsageSample = { + ts: string + fiveHour?: WindowSample + sevenDay?: WindowSample + sevenDayOpus?: WindowSample + sevenDaySonnet?: WindowSample + scoped?: Array<{ label: string; pct: number; resetsAt?: string }> +} + +function cacheDir(): string { + return process.env['CODEBURN_CACHE_DIR'] ?? join(homedir(), '.cache', 'codeburn') +} + +export function usageSamplesPath(): string { + return join(cacheDir(), 'usage-samples.jsonl') +} + +type OauthRecord = { accessToken?: string; expiresAt?: number } + +function parseCredentialJson(text: string): OauthRecord | undefined { + try { + const root = JSON.parse(text) as { claudeAiOauth?: OauthRecord } + return root.claudeAiOauth + } catch { + return undefined + } +} + +/** Claude Code's OAuth access token: the credentials file where it exists + * (Linux, some macOS setups), else the macOS keychain item Claude Code + * writes. Returns undefined when absent or expired — never throws. */ +export async function readClaudeAccessToken(): Promise { + const fromFile = await readFile(join(homedir(), '.claude', '.credentials.json'), 'utf8') + .then(parseCredentialJson) + .catch(() => undefined) + const candidates: OauthRecord[] = fromFile ? [fromFile] : [] + + if (candidates.length === 0 && process.platform === 'darwin') { + const fromKeychain = await execFileAsync( + 'security', ['find-generic-password', '-s', KEYCHAIN_SERVICE, '-w'], + { timeout: 5000 }, + ) + .then(r => parseCredentialJson(r.stdout)) + .catch(() => undefined) + if (fromKeychain) candidates.push(fromKeychain) + } + + for (const oauth of candidates) { + const token = oauth.accessToken?.trim() + if (!token) continue + // expiresAt is epoch millis; skip tokens already (about to be) expired. + if (oauth.expiresAt !== undefined && oauth.expiresAt < Date.now() + 60_000) continue + return token + } + return undefined +} + +type UsageResponse = { + five_hour?: { utilization?: number; resets_at?: string } + seven_day?: { utilization?: number; resets_at?: string } + seven_day_opus?: { utilization?: number; resets_at?: string } + seven_day_sonnet?: { utilization?: number; resets_at?: string } + limits?: Array<{ + kind?: string + percent?: number + resets_at?: string + scope?: { model?: { display_name?: string } } + }> +} + +function window(w?: { utilization?: number; resets_at?: string }): WindowSample | undefined { + if (w?.utilization === undefined || !Number.isFinite(w.utilization)) return undefined + return { pct: w.utilization, ...(w.resets_at ? { resetsAt: w.resets_at } : {}) } +} + +export function parseUsageResponse(body: string, ts: Date): UsageSample | undefined { + let r: UsageResponse + try { + r = JSON.parse(body) as UsageResponse + } catch { + return undefined + } + const scoped = (r.limits ?? []).flatMap(limit => { + if (limit.kind !== 'weekly_scoped') return [] + const label = limit.scope?.model?.display_name + if (!label || limit.percent === undefined || !Number.isFinite(limit.percent)) return [] + return [{ label, pct: limit.percent, ...(limit.resets_at ? { resetsAt: limit.resets_at } : {}) }] + }) + const sample: UsageSample = { + ts: ts.toISOString(), + ...(window(r.five_hour) ? { fiveHour: window(r.five_hour) } : {}), + ...(window(r.seven_day) ? { sevenDay: window(r.seven_day) } : {}), + ...(window(r.seven_day_opus) ? { sevenDayOpus: window(r.seven_day_opus) } : {}), + ...(window(r.seven_day_sonnet) ? { sevenDaySonnet: window(r.seven_day_sonnet) } : {}), + ...(scoped.length > 0 ? { scoped } : {}), + } + // A sample with no windows at all carries no signal; don't record it. + const { ts: _, ...windows } = sample + return Object.keys(windows).length > 0 ? sample : undefined +} + +/** True when the samples file was written recently enough that another sample + * would add noise, not signal. Missing file means sample away. */ +export async function sampledRecently(now = Date.now()): Promise { + const s = await stat(usageSamplesPath()).catch(() => undefined) + return s !== undefined && now - s.mtimeMs < SAMPLE_MIN_INTERVAL_MS +} + +export type SampleOutcome = + | { ok: true; sample: UsageSample } + | { ok: false; reason: 'throttled' | 'no-token' | 'http-error' | 'malformed' | 'network' } + +/** One sample: token → endpoint → append. Every failure is a quiet, typed + * outcome; the sampler must never break the command it rides along with. */ +export async function sampleUsageNow(opts: { force?: boolean } = {}): Promise { + if (!opts.force && await sampledRecently()) return { ok: false, reason: 'throttled' } + const token = await readClaudeAccessToken() + if (!token) return { ok: false, reason: 'no-token' } + + let body: string + try { + const res = await fetch(USAGE_URL, { + headers: { + 'Authorization': `Bearer ${token}`, + 'anthropic-beta': BETA_HEADER, + 'User-Agent': USER_AGENT, + 'Accept': 'application/json', + }, + signal: AbortSignal.timeout(10_000), + }) + if (!res.ok) return { ok: false, reason: 'http-error' } + body = await res.text() + } catch { + return { ok: false, reason: 'network' } + } + + const sample = parseUsageResponse(body, new Date()) + if (!sample) return { ok: false, reason: 'malformed' } + + await mkdir(cacheDir(), { recursive: true }) + await appendFile(usageSamplesPath(), JSON.stringify(sample) + '\n', 'utf8') + return { ok: true, sample } +} + +export type SamplesInfo = { count: number; firstTs?: string; lastTs?: string } + +export async function readSamplesInfo(): Promise { + const text = await readFile(usageSamplesPath(), 'utf8').catch(() => '') + const lines = text.split('\n').filter(l => l.trim().length > 0) + if (lines.length === 0) return { count: 0 } + const first = parseTs(lines[0]!) + const last = parseTs(lines[lines.length - 1]!) + return { + count: lines.length, + ...(first ? { firstTs: first } : {}), + ...(last ? { lastTs: last } : {}), + } +} + +function parseTs(line: string): string | undefined { + try { + const parsed = JSON.parse(line) as { ts?: string } + return typeof parsed.ts === 'string' ? parsed.ts : undefined + } catch { + return undefined + } +} diff --git a/tests/usage-sampler.test.ts b/tests/usage-sampler.test.ts new file mode 100644 index 00000000..ede730b3 --- /dev/null +++ b/tests/usage-sampler.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it, beforeEach, afterEach } from 'vitest' +import { mkdtemp, rm, writeFile, utimes } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + parseUsageResponse, + readSamplesInfo, + sampledRecently, + usageSamplesPath, + SAMPLE_MIN_INTERVAL_MS, +} from '../src/usage-sampler.js' + +const TS = new Date('2026-08-12T12:00:00Z') + +// Captured shape of the oauth/usage endpoint (mirrors the menubar's decoder). +const RESPONSE = JSON.stringify({ + five_hour: { utilization: 37, resets_at: '2026-08-12T15:00:00Z' }, + seven_day: { utilization: 90, resets_at: '2026-08-14T00:00:00Z' }, + seven_day_opus: { utilization: 12 }, + limits: [ + { kind: 'weekly_scoped', percent: 100, resets_at: '2026-08-14T00:00:00Z', scope: { model: { display_name: 'Fable 5' } } }, + { kind: 'something_else', percent: 5 }, + ], +}) + +describe('parseUsageResponse', () => { + it('maps windows and model-scoped weekly limits', () => { + const sample = parseUsageResponse(RESPONSE, TS) + expect(sample).toEqual({ + ts: TS.toISOString(), + fiveHour: { pct: 37, resetsAt: '2026-08-12T15:00:00Z' }, + sevenDay: { pct: 90, resetsAt: '2026-08-14T00:00:00Z' }, + sevenDayOpus: { pct: 12 }, + scoped: [{ label: 'Fable 5', pct: 100, resetsAt: '2026-08-14T00:00:00Z' }], + }) + }) + + it('rejects malformed bodies and empty responses', () => { + expect(parseUsageResponse('not json', TS)).toBeUndefined() + expect(parseUsageResponse('{}', TS)).toBeUndefined() + expect(parseUsageResponse(JSON.stringify({ five_hour: {} }), TS)).toBeUndefined() + }) +}) + +describe('sampling state on disk', () => { + let dir: string + let prevCacheDir: string | undefined + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'codeburn-sampler-')) + prevCacheDir = process.env['CODEBURN_CACHE_DIR'] + process.env['CODEBURN_CACHE_DIR'] = dir + }) + + afterEach(async () => { + if (prevCacheDir === undefined) delete process.env['CODEBURN_CACHE_DIR'] + else process.env['CODEBURN_CACHE_DIR'] = prevCacheDir + await rm(dir, { recursive: true, force: true }) + }) + + it('throttles on a fresh samples file, allows on a stale or missing one', async () => { + expect(await sampledRecently()).toBe(false) + + await writeFile(usageSamplesPath(), '{"ts":"2026-08-12T11:59:00Z"}\n') + expect(await sampledRecently()).toBe(true) + + const stale = new Date(Date.now() - SAMPLE_MIN_INTERVAL_MS - 1000) + await utimes(usageSamplesPath(), stale, stale) + expect(await sampledRecently()).toBe(false) + }) + + it('summarizes recorded samples', async () => { + expect(await readSamplesInfo()).toEqual({ count: 0 }) + + await writeFile( + usageSamplesPath(), + '{"ts":"2026-08-12T10:00:00Z"}\n{"ts":"2026-08-12T11:00:00Z"}\n', + ) + expect(await readSamplesInfo()).toEqual({ + count: 2, + firstTs: '2026-08-12T10:00:00Z', + lastTs: '2026-08-12T11:00:00Z', + }) + }) +})