diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index c0e0dd2e96..fa6a5f143f 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "84c52421265e9fc8" + static let value = "4169e0ab9fbb53c2" } diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageAccountingSelector.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageAccountingSelector.swift new file mode 100644 index 0000000000..37e6b385fc --- /dev/null +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageAccountingSelector.swift @@ -0,0 +1,108 @@ +import Foundation + +enum CodexLineageAccountingMode: String, CaseIterable, Sendable { + case legacy + case shadow + case lineage + + static let defaultMode: Self = .legacy + static let schemaVersion = 1 + + var producerKeySuffix: String { + "lineage-accounting:\(self.rawValue):v\(Self.schemaVersion)" + } +} + +/// Selects one whole-scan authority while preserving a permanent family-scoped containment route. +enum CodexLineageAccountingSelector { + typealias PackedDays = [String: [String: [Int]]] + + enum SelectionError: Error, Equatable { + case missingContainedFamilyEvidence + } + + struct ContainedDocument: Equatable, Sendable { + /// Stable logical rollout identity. Physical active/archive copies share this value. + let identity: String + let days: PackedDays + } + + struct ContainedFamily: Equatable, Sendable { + /// Legacy file contributions belonging only to this contained family. + let documents: [ContainedDocument] + } + + struct Selection: Equatable, Sendable { + let days: PackedDays + let usedLineageAuthority: Bool + let containedFamilyCount: Int + } + + static func select( + mode: CodexLineageAccountingMode, + legacyDays: PackedDays, + primaryRows: [CodexLineageLedger.DailyRow], + containedFamilies: [ContainedFamily]) -> Selection + { + guard mode == .lineage else { + return Selection(days: legacyDays, usedLineageAuthority: false, containedFamilyCount: 0) + } + var days = Self.days(from: primaryRows) + for family in containedFamilies { + Self.add(Self.containedDays(family.documents), to: &days) + } + return Selection( + days: days, + usedLineageAuthority: true, + containedFamilyCount: containedFamilies.count) + } + + /// A contained family contributes once. Physical copies of one logical rollout use an envelope; + /// distinct siblings remain additive so containment does not discard their independent work. + private static func containedDays(_ documents: [ContainedDocument]) -> PackedDays { + var documentsByIdentity: [String: [PackedDays]] = [:] + for document in documents { + documentsByIdentity[document.identity, default: []].append(document.days) + } + var result: PackedDays = [:] + for copies in documentsByIdentity.values { + var envelope: PackedDays = [:] + for copy in copies { + for (day, models) in copy { + for (model, packed) in models { + let existing = envelope[day]?[model] ?? [0, 0, 0] + envelope[day, default: [:]][model] = (0..<3).map { index in + max(existing[safe: index] ?? 0, packed[safe: index] ?? 0) + } + } + } + } + Self.add(envelope, to: &result) + } + return result + } + + private static func days(from rows: [CodexLineageLedger.DailyRow]) -> PackedDays { + var days: PackedDays = [:] + for row in rows { + var packed = days[row.day]?[row.model] ?? [0, 0, 0] + packed[0] += row.totals.input + packed[1] += row.totals.cached + packed[2] += row.totals.output + days[row.day, default: [:]][row.model] = packed + } + return days + } + + private static func add(_ source: PackedDays, to destination: inout PackedDays) { + for (day, models) in source { + for (model, packed) in models { + var value = destination[day]?[model] ?? [0, 0, 0] + for index in 0..<3 { + value[index] += packed[safe: index] ?? 0 + } + destination[day, default: [:]][model] = value + } + } + } +} diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 88f724d30e..bb8f024a78 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -28,6 +28,7 @@ enum CostUsageScanner { var codexTraceDatabaseURL: URL? var refreshMinIntervalSeconds: TimeInterval = 60 var claudeLogProviderFilter: ClaudeLogProviderFilter = .all + var codexLineageAccountingMode: CodexLineageAccountingMode = .defaultMode /// Force a full rescan, ignoring per-file cache and incremental offsets. var forceRescan: Bool = false @@ -37,6 +38,7 @@ enum CostUsageScanner { cacheRoot: URL? = nil, codexTraceDatabaseURL: URL? = nil, claudeLogProviderFilter: ClaudeLogProviderFilter = .all, + codexLineageAccountingMode: CodexLineageAccountingMode = .defaultMode, forceRescan: Bool = false) { self.codexSessionsRoot = codexSessionsRoot @@ -44,6 +46,7 @@ enum CostUsageScanner { self.cacheRoot = cacheRoot self.codexTraceDatabaseURL = codexTraceDatabaseURL self.claudeLogProviderFilter = claudeLogProviderFilter + self.codexLineageAccountingMode = codexLineageAccountingMode self.forceRescan = forceRescan } } @@ -2613,7 +2616,11 @@ enum CostUsageScanner { options: Options, checkCancellation: CancellationCheck?) throws -> CostUsageDailyReport { - var cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: options.cacheRoot) + let accountingProducerKey = Self.codexAccountingProducerKey(mode: options.codexLineageAccountingMode) + var cache = CostUsageCacheIO.load( + provider: .codex, + cacheRoot: options.cacheRoot, + producerKey: accountingProducerKey) let nowMs = Int64(now.timeIntervalSince1970 * 1000) let plan = Self.makeCodexRefreshPlan(cache: cache, range: range, now: now, nowMs: nowMs, options: options) @@ -2740,12 +2747,13 @@ enum CostUsageScanner { ? [cachedUntilKey, range.scanUntilKey].compactMap(\.self).max() ?? range.scanUntilKey : range.scanUntilKey Self.pruneDays(cache: &cache, sinceKey: retainedSinceKey, untilKey: retainedUntilKey) - try Self.recordCodexLineageShadow( + try Self.applyCodexLineageAccounting( + mode: options.codexLineageAccountingMode, files: files, roots: plan.roots, - cache: cache, - range: range, + cache: &cache, checkCancellation: checkCancellation) + Self.pruneDays(cache: &cache, sinceKey: retainedSinceKey, untilKey: retainedUntilKey) cache.roots = plan.rootsFingerprint cache.scanSinceKey = retainedSinceKey cache.scanUntilKey = retainedUntilKey @@ -2768,7 +2776,11 @@ enum CostUsageScanner { } cache.lastScanUnixMs = nowMs try checkCancellation?() - CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: options.cacheRoot) + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: options.cacheRoot, + producerKey: accountingProducerKey) } return Self.buildCodexReportFromCache( @@ -2779,48 +2791,93 @@ enum CostUsageScanner { priorityTurns: plan.priorityTurns) } - private static func recordCodexLineageShadow( + static func codexAccountingProducerKey(mode: CodexLineageAccountingMode) -> String { + let base = CostUsageCacheIO.currentProducerKey(provider: .codex) ?? "codex" + return mode == .legacy ? base : base + ":" + mode.producerKeySuffix + } + + static func shouldRunCodexLineage(mode: CodexLineageAccountingMode) -> Bool { + mode != .legacy + } + + private static func applyCodexLineageAccounting( + mode: CodexLineageAccountingMode, files: [URL], roots: [URL], - cache: CostUsageCache, - range: CostUsageDayRange, + cache: inout CostUsageCache, checkCancellation: CancellationCheck?) throws { + guard self.shouldRunCodexLineage(mode: mode) else { return } do { - let report = try CodexLineageShadow.run( + let discovery = try CodexLineageTwoPassDiscovery.discover( includedFiles: files, roots: roots, - legacyDays: cache.days, - dayRange: range.sinceKey...range.untilKey, + checkCancellation: checkCancellation) + let families = try CodexLineageEngine.prepareDescriptorFamilies( + descriptors: discovery.descriptors, + unresolvedParents: discovery.unresolvedParents, + checkCancellation: checkCancellation) + let lineage = try CodexLineageEngine.reconcileStreaming( + families: families, localTimeZone: .current, checkCancellation: checkCancellation) - self.log.info( - "Codex lineage shadow comparison completed", + let resultsByStableID = Dictionary(uniqueKeysWithValues: lineage.families.map { ($0.stableID, $0) }) + let contained = families.compactMap { family -> CodexLineageAccountingSelector.ContainedFamily? in + guard let result = resultsByStableID[family.stableID], case .contained = result.quality else { + return nil + } + let documents = family.descriptors.compactMap { descriptor -> + CodexLineageAccountingSelector.ContainedDocument? in + guard let days = cache.files[descriptor.fileURL.path]?.days else { return nil } + let identity = descriptor.scopeID + "\u{0}" + + (descriptor.metadataSessionID ?? descriptor.ownerID).lowercased() + return .init(identity: identity, days: days) + } + guard documents.count == family.descriptors.count else { return nil } + return .init(documents: documents) + } + let expectedContainedCount = lineage.families.count { + if case .contained = $0.quality { + return true + } + return false + } + if mode == .lineage, contained.count != expectedContainedCount { + throw CodexLineageAccountingSelector.SelectionError.missingContainedFamilyEvidence + } + let selection = CodexLineageAccountingSelector.select( + mode: mode, + legacyDays: cache.days, + primaryRows: lineage.report.localRows, + containedFamilies: contained) + try checkCancellation?() + + #if DEBUG + self.log.debug( + "Codex lineage accounting comparison completed", metadata: [ - "accepted": "\(report.acceptedObservationCount)", - "components": "\(report.componentCount)", - "days": "\(report.days.count)", - "duplicates": "\(report.duplicateObservationCount)", - "referencedParents": "\(report.referencedParentDocumentCount)", - "rejected": "\(report.rejectedObservationCount)", - "unresolvedParents": "\(report.unresolvedParentCount)", + "containedFamilies": "\(contained.count)", + "families": "\(lineage.diagnostics.familyCount)", + "mode": mode.rawValue, + "recomputedFamilies": "\(lineage.diagnostics.recomputedFamilyCount)", ]) - for day in report.days where day.delta != .zero { - self.log.info( - "Codex lineage shadow daily difference", - metadata: [ - "cachedDelta": "\(day.delta.cached)", - "day": "\(day.day)", - "inputDelta": "\(day.delta.input)", - "outputDelta": "\(day.delta.output)", - ]) + #endif + + if selection.usedLineageAuthority { + cache.days = selection.days + // Legacy per-file rows are not valid pricing/project evidence for ledger totals. + // Dropping them also makes rollback and the next mode-specific refresh rebuild safely. + cache.files = [:] } } catch is CancellationError { throw CancellationError() } catch { - self.log.warning( - "Codex lineage shadow comparison failed", - metadata: ["errorType": "\(type(of: error))"]) + if mode == .lineage { + throw error + } + #if DEBUG + self.log.debug("Codex lineage shadow comparison failed; legacy totals remain authoritative") + #endif } } diff --git a/Tests/CodexBarTests/CodexLineageAccountingSelectorTests.swift b/Tests/CodexBarTests/CodexLineageAccountingSelectorTests.swift new file mode 100644 index 0000000000..c1d75429f4 --- /dev/null +++ b/Tests/CodexBarTests/CodexLineageAccountingSelectorTests.swift @@ -0,0 +1,125 @@ +import Testing +@testable import CodexBarCore + +struct CodexLineageAccountingSelectorTests { + @Test + func `legacy and shadow modes keep legacy authority`() { + let legacy = Self.days(input: 100) + let primary = [Self.row(input: 40)] + for mode in [CodexLineageAccountingMode.legacy, .shadow] { + let selection = CodexLineageAccountingSelector.select( + mode: mode, + legacyDays: legacy, + primaryRows: primary, + containedFamilies: [.init(documents: [.init(identity: "parent", days: Self.days(input: 20))])]) + #expect(selection.days == legacy) + #expect(!selection.usedLineageAuthority) + #expect(selection.containedFamilyCount == 0) + } + } + + @Test + func `lineage mode combines primary and family containment exactly once`() { + let selection = CodexLineageAccountingSelector.select( + mode: .lineage, + legacyDays: Self.days(input: 999), + primaryRows: [Self.row(input: 100)], + containedFamilies: [ + .init(documents: [ + .init(identity: "copy", days: Self.days(input: 40)), + .init(identity: "copy", days: Self.days(input: 40)), + ]), + .init(documents: [.init(identity: "other", days: Self.days(input: 10))]), + ]) + + #expect(selection.usedLineageAuthority) + #expect(selection.containedFamilyCount == 2) + #expect(selection.days["2026-07-09"]?["gpt-5.4"]?[0] == 150) + } + + @Test + func `contained family uses component envelope instead of summing fork copies`() { + let first: CodexLineageAccountingSelector.PackedDays = [ + "2026-07-09": ["gpt-5.4": [100, 80, 5]], + ] + let copied: CodexLineageAccountingSelector.PackedDays = [ + "2026-07-09": ["gpt-5.4": [100, 60, 8]], + ] + let selection = CodexLineageAccountingSelector.select( + mode: .lineage, + legacyDays: [:], + primaryRows: [], + containedFamilies: [.init(documents: [ + .init(identity: "same", days: first), + .init(identity: "same", days: copied), + ])]) + + #expect(selection.days["2026-07-09"]?["gpt-5.4"] == [100, 80, 8]) + } + + @Test + func `contained siblings remain additive while physical copies are enveloped`() { + let selection = CodexLineageAccountingSelector.select( + mode: .lineage, + legacyDays: [:], + primaryRows: [], + containedFamilies: [.init(documents: [ + .init(identity: "child-a", days: Self.days(input: 40)), + .init(identity: "child-a", days: Self.days(input: 40)), + .init(identity: "child-b", days: Self.days(input: 10)), + ])]) + + #expect(selection.days["2026-07-09"]?["gpt-5.4"]?[0] == 50) + } + + @Test + func `mode cache suffixes are schema scoped and distinct`() { + #expect(CodexLineageAccountingMode.defaultMode == .legacy) + #expect(CodexLineageAccountingMode.shadow.producerKeySuffix != CodexLineageAccountingMode.lineage + .producerKeySuffix) + #expect(CodexLineageAccountingMode.shadow.producerKeySuffix.contains("v1")) + } + + @Test + func `scanner defaults to legacy and skips lineage execution`() { + let options = CostUsageScanner.Options() + #expect(options.codexLineageAccountingMode == .legacy) + #expect(!CostUsageScanner.shouldRunCodexLineage(mode: options.codexLineageAccountingMode)) + #expect(CostUsageScanner.shouldRunCodexLineage(mode: .shadow)) + #expect(CostUsageScanner.shouldRunCodexLineage(mode: .lineage)) + } + + @Test + func `mode specific producer key mismatch invalidates cache for rollback rebuild`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let shadowKey = CostUsageScanner.codexAccountingProducerKey(mode: .shadow) + let legacyKey = CostUsageScanner.codexAccountingProducerKey(mode: .legacy) + var shadowCache = CostUsageCache() + shadowCache.days = Self.days(input: 100) + CostUsageCacheIO.save( + provider: .codex, + cache: shadowCache, + cacheRoot: environment.cacheRoot, + producerKey: shadowKey) + + let rollbackLoad = CostUsageCacheIO.load( + provider: .codex, + cacheRoot: environment.cacheRoot, + producerKey: legacyKey) + #expect(rollbackLoad.days.isEmpty) + #expect(shadowKey != legacyKey) + } + + private static func days(input: Int) -> CodexLineageAccountingSelector.PackedDays { + ["2026-07-09": ["gpt-5.4": [input, 0, 0]]] + } + + private static func row(input: Int) -> CodexLineageLedger.DailyRow { + .init( + day: "2026-07-09", + model: "gpt-5.4", + totals: .init(input: input, cached: 0, output: 0), + costUSD: 0) + } +}