From 19d63cb10c5cfb66319bfa38806b00273bc2f909 Mon Sep 17 00:00:00 2001 From: Bryan Font Date: Mon, 13 Jul 2026 13:39:06 -0400 Subject: [PATCH] Run Codex lineage accounting in shadow --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Providers/Codex/CodexLineageLedger.swift | 11 ++- .../Providers/Codex/CodexLineageShadow.swift | 94 +++++++++++++++++++ .../Vendored/CostUsage/CostUsageScanner.swift | 55 ++++++++++- .../CodexLineageShadowTests.swift | 68 ++++++++++++++ 5 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift create mode 100644 Tests/CodexBarTests/CodexLineageShadowTests.swift diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index b4eeb35463..d57f41412c 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "a231538286ac81c9" + static let value = "07457314442f300f" } diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift index a2053a539a..bf59ca377a 100644 --- a/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift @@ -69,9 +69,14 @@ enum CodexLineageLedger { case invalidTimestamp(String) } - static func reconcile(documents: [Document], localTimeZone: TimeZone) throws -> Report { + static func reconcile( + documents: [Document], + localTimeZone: TimeZone, + checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> Report + { var graph = DisjointSet() for document in documents { + try checkCancellation?() guard !document.ownerID.isEmpty else { throw LedgerError.emptyOwnerID } graph.insert(document.ownerID) if let metadataSessionID = Self.nonEmpty(document.metadataSessionID) { @@ -85,9 +90,11 @@ enum CodexLineageLedger { var acceptedByComponent: [String: [Fingerprint: AcceptedObservation]] = [:] var physicalObservationCount = 0 for document in documents { + try checkCancellation?() let componentID = graph.find(document.ownerID) var accepted = acceptedByComponent[componentID] ?? [:] for observation in document.observations { + try checkCancellation?() physicalObservationCount += 1 let date = try Self.date(from: observation.timestamp) let fingerprint = Fingerprint(last: observation.last, total: observation.total) @@ -115,8 +122,10 @@ enum CodexLineageLedger { var localRows: [DailyRowKey: DailyRowValue] = [:] var acceptedObservationCount = 0 for accepted in acceptedByComponent.values { + try checkCancellation?() acceptedObservationCount += accepted.count for observation in accepted.values { + try checkCancellation?() Self.add(observation.last, on: observation.date, timeZone: .gmt, to: &utcDays) Self.add(observation.last, on: observation.date, timeZone: localTimeZone, to: &localDays) Self.addRow(observation, timeZone: .gmt, to: &utcRows) diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift new file mode 100644 index 0000000000..a6938113e0 --- /dev/null +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift @@ -0,0 +1,94 @@ +import Foundation + +/// Privacy-safe comparison between the production file scanner and the experimental lineage ledger. +enum CodexLineageShadow { + struct DayDifference: Equatable, Sendable { + let day: String + let legacy: CodexLineageLedger.Totals + let ledger: CodexLineageLedger.Totals + + var delta: CodexLineageLedger.Totals { + .init( + input: self.ledger.input - self.legacy.input, + cached: self.ledger.cached - self.legacy.cached, + output: self.ledger.output - self.legacy.output) + } + } + + struct Report: Equatable, Sendable { + let days: [DayDifference] + let acceptedObservationCount: Int + let duplicateObservationCount: Int + let componentCount: Int + let referencedParentDocumentCount: Int + let unresolvedParentCount: Int + let rejectedObservationCount: Int + } + + static func run( + includedFiles: [URL], + roots: [URL], + legacyDays: [String: [String: [Int]]], + dayRange: ClosedRange, + localTimeZone: TimeZone, + checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> Report + { + let discovery = try CodexLineageDiscovery.discover( + includedFiles: includedFiles, + roots: roots, + checkCancellation: checkCancellation) + var rejectedObservationCount = 0 + var documents: [CodexLineageLedger.Document] = [] + documents.reserveCapacity(discovery.documents.count) + for document in discovery.documents { + try checkCancellation?() + var observations: [CodexLineageLedger.Observation] = [] + observations.reserveCapacity(document.observations.count) + for observation in document.observations { + try checkCancellation?() + let accepted = CostUsageScanner.dateFromTimestamp(observation.timestamp) != nil + if !accepted { + rejectedObservationCount += 1 + } else { + observations.append(observation) + } + } + documents.append(CodexLineageLedger.Document( + ownerID: document.ownerID, + metadataSessionID: document.metadataSessionID, + parentSessionID: document.parentSessionID, + observations: observations)) + } + let ledger = try CodexLineageLedger.reconcile( + documents: documents, + localTimeZone: localTimeZone, + checkCancellation: checkCancellation) + let legacy = Self.legacyTotalsByDay(legacyDays) + let dayKeys = Set(legacy.keys).union(ledger.localDays.keys) + .filter(dayRange.contains) + .sorted() + let days = dayKeys.map { day in + DayDifference(day: day, legacy: legacy[day] ?? .zero, ledger: ledger.localDays[day] ?? .zero) + } + return Report( + days: days, + acceptedObservationCount: ledger.acceptedObservationCount, + duplicateObservationCount: ledger.duplicateObservationCount, + componentCount: ledger.componentCount, + referencedParentDocumentCount: discovery.referencedParentDocumentCount, + unresolvedParentCount: discovery.unresolvedParentIDs.count, + rejectedObservationCount: rejectedObservationCount) + } + + private static func legacyTotalsByDay( + _ days: [String: [String: [Int]]]) -> [String: CodexLineageLedger.Totals] + { + days.mapValues { models in + models.values.reduce(into: CodexLineageLedger.Totals.zero) { totals, packed in + totals.input += packed[safe: 0] ?? 0 + totals.cached += packed[safe: 1] ?? 0 + totals.output += packed[safe: 2] ?? 0 + } + } + } +} diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index aa7f0cc51b..08c25b83e3 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -1704,8 +1704,7 @@ enum CostUsageScanner { warnedAboutUnparsedTimestamp = true self.log.warning( "Codex cost usage could not parse parent token snapshot timestamp; " - + "falling back to lexical comparison", - metadata: ["path": fileURL.path, "timestamp": timestamp]) + + "falling back to lexical comparison") } return date } @@ -2543,6 +2542,7 @@ enum CostUsageScanner { shouldRefresh: shouldRefresh) } + // swiftlint:disable:next function_body_length private static func loadCodexDaily( range: CostUsageDayRange, now: Date, @@ -2676,6 +2676,12 @@ enum CostUsageScanner { ? [cachedUntilKey, range.scanUntilKey].compactMap(\.self).max() ?? range.scanUntilKey : range.scanUntilKey Self.pruneDays(cache: &cache, sinceKey: retainedSinceKey, untilKey: retainedUntilKey) + try Self.recordCodexLineageShadow( + files: files, + roots: plan.roots, + cache: cache, + range: range, + checkCancellation: checkCancellation) cache.roots = plan.rootsFingerprint cache.scanSinceKey = retainedSinceKey cache.scanUntilKey = retainedUntilKey @@ -2709,6 +2715,51 @@ enum CostUsageScanner { priorityTurns: plan.priorityTurns) } + private static func recordCodexLineageShadow( + files: [URL], + roots: [URL], + cache: CostUsageCache, + range: CostUsageDayRange, + checkCancellation: CancellationCheck?) throws + { + do { + let report = try CodexLineageShadow.run( + includedFiles: files, + roots: roots, + legacyDays: cache.days, + dayRange: range.sinceKey...range.untilKey, + localTimeZone: .current, + checkCancellation: checkCancellation) + self.log.info( + "Codex lineage shadow comparison completed", + metadata: [ + "accepted": "\(report.acceptedObservationCount)", + "components": "\(report.componentCount)", + "days": "\(report.days.count)", + "duplicates": "\(report.duplicateObservationCount)", + "referencedParents": "\(report.referencedParentDocumentCount)", + "rejected": "\(report.rejectedObservationCount)", + "unresolvedParents": "\(report.unresolvedParentCount)", + ]) + for day in report.days where day.delta != .zero { + self.log.info( + "Codex lineage shadow daily difference", + metadata: [ + "cachedDelta": "\(day.delta.cached)", + "day": "\(day.day)", + "inputDelta": "\(day.delta.input)", + "outputDelta": "\(day.delta.output)", + ]) + } + } catch is CancellationError { + throw CancellationError() + } catch { + self.log.warning( + "Codex lineage shadow comparison failed", + metadata: ["errorType": "\(type(of: error))"]) + } + } + private static func codexFileScanContext( range: CostUsageDayRange, options: Options, diff --git a/Tests/CodexBarTests/CodexLineageShadowTests.swift b/Tests/CodexBarTests/CodexLineageShadowTests.swift new file mode 100644 index 0000000000..51eb59adfc --- /dev/null +++ b/Tests/CodexBarTests/CodexLineageShadowTests.swift @@ -0,0 +1,68 @@ +import Foundation +import Testing +@testable import CodexBarCore + +struct CodexLineageShadowTests { + @Test + func `shadow comparison records deltas and diagnostics without changing legacy totals`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let parentID = "11111111-1111-4111-8111-111111111111" + let childID = "22222222-2222-4222-8222-222222222222" + let parent = try Self.writeRollout( + root: environment.codexSessionsRoot, + ownerID: parentID, + metadataID: parentID, + events: [Self.event(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100)]) + let child = try Self.writeRollout( + root: environment.codexSessionsRoot, + ownerID: childID, + metadataID: childID, + parentID: parentID, + events: [ + Self.event(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100), + Self.event(timestamp: "not-a-time", input: 50, totalInput: 150), + ]) + let legacyDays = ["2026-07-09": ["gpt-5.4": [150, 20, 10]]] + + let report = try CodexLineageShadow.run( + includedFiles: [parent, child], + roots: [environment.codexSessionsRoot], + legacyDays: legacyDays, + dayRange: "2026-07-09"..."2026-07-09", + localTimeZone: .gmt) + + #expect(legacyDays["2026-07-09"]?["gpt-5.4"] == [150, 20, 10]) + #expect(report.days == [.init( + day: "2026-07-09", + legacy: .init(input: 150, cached: 20, output: 10), + ledger: .init(input: 100, cached: 0, output: 0))]) + #expect(report.days[0].delta == .init(input: -50, cached: -20, output: -10)) + #expect(report.acceptedObservationCount == 1) + #expect(report.duplicateObservationCount == 1) + #expect(report.componentCount == 1) + #expect(report.rejectedObservationCount == 1) + #expect(report.unresolvedParentCount == 0) + } + + private static func writeRollout( + root: URL, + ownerID: String, + metadataID: String, + parentID: String? = nil, + events: [String]) throws -> URL + { + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let file = root.appendingPathComponent("rollout-2026-07-09T12-00-00-\(ownerID).jsonl") + let parent = parentID.map { #", "forked_from_id":"\#($0)""# } ?? "" + let metadata = #"{"type":"session_meta","payload":{"id":"\#(metadataID)"\#(parent)}}"# + try ([metadata] + events).joined(separator: "\n").write(to: file, atomically: true, encoding: .utf8) + return file + } + + private static func event(timestamp: String, input: Int, totalInput: Int) -> String { + #"{"type":"event_msg","timestamp":"\#(timestamp)","payload":{"type":"token_count","info":{"# + + #""last_token_usage":{"input_tokens":\#(input),"cached_input_tokens":0,"output_tokens":0},"# + + #""total_token_usage":{"input_tokens":\#(totalInput),"cached_input_tokens":0,"output_tokens":0}}}}"# + } +}