diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 2a1047a65a..a27b750991 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "865e101428b49ab7" + static let value = "f1fe17eb233c9f4d" } diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift new file mode 100644 index 0000000000..0b75a82e87 --- /dev/null +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift @@ -0,0 +1,137 @@ +import Foundation + +/// Expands an already bounded rollout-file selection with only the ancestors its documents reference. +enum CodexLineageDiscovery { + struct Report: Equatable, Sendable { + let documents: [CodexLineageLedger.Document] + let referencedParentDocumentCount: Int + let unresolvedParentIDs: Set + } + + static func discover( + includedFiles: [URL], + roots: [URL], + checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> Report + { + var locator = ParentFileLocator(roots: roots, checkCancellation: checkCancellation) + var documents: [CodexLineageLedger.Document] = [] + var knownIDs: Set = [] + var seenPaths: Set = [] + var pendingParentIDs: [String] = [] + var unresolvedParentIDs: Set = [] + var referencedParentDocumentCount = 0 + + func remember(_ document: CodexLineageLedger.Document) { + documents.append(document) + knownIDs.insert(Self.canonicalSessionID(document.ownerID)) + if let metadataSessionID = Self.nonEmpty(document.metadataSessionID) { + knownIDs.insert(Self.canonicalSessionID(metadataSessionID)) + } + if let parentSessionID = Self.nonEmpty(document.parentSessionID) { + pendingParentIDs.append(Self.canonicalSessionID(parentSessionID)) + } + } + + for fileURL in includedFiles.sorted(by: { $0.path < $1.path }) { + try checkCancellation?() + let path = fileURL.standardizedFileURL.path + guard seenPaths.insert(path).inserted else { continue } + try remember(CostUsageScanner.parseCodexLineageDocument( + fileURL: fileURL, + checkCancellation: checkCancellation)) + } + + var nextParent = 0 + while nextParent < pendingParentIDs.count { + try checkCancellation?() + let parentID = pendingParentIDs[nextParent] + nextParent += 1 + guard !knownIDs.contains(parentID), !unresolvedParentIDs.contains(parentID) else { continue } + guard let parentURL = try locator.fileURL(for: parentID) else { + unresolvedParentIDs.insert(parentID) + continue + } + let path = parentURL.standardizedFileURL.path + guard seenPaths.insert(path).inserted else { + unresolvedParentIDs.insert(parentID) + continue + } + let parent = try CostUsageScanner.parseCodexLineageDocument( + fileURL: parentURL, + checkCancellation: checkCancellation) + let ownerID = Self.canonicalSessionID(parent.ownerID) + let metadataSessionID = parent.metadataSessionID.map(Self.canonicalSessionID) + guard ownerID == parentID || metadataSessionID == parentID else { + unresolvedParentIDs.insert(parentID) + continue + } + referencedParentDocumentCount += 1 + remember(parent) + } + + return Report( + documents: documents, + referencedParentDocumentCount: referencedParentDocumentCount, + unresolvedParentIDs: unresolvedParentIDs) + } + + private static func nonEmpty(_ value: String?) -> String? { + guard let value, !value.isEmpty else { return nil } + return value + } + + private static func canonicalSessionID(_ value: String) -> String { + UUID(uuidString: value)?.uuidString.lowercased() ?? value + } + + private struct ParentFileLocator { + let roots: [URL] + let checkCancellation: CostUsageScanner.CancellationCheck? + var didIndexRoots = false + var filesByID: [String: URL] = [:] + + mutating func fileURL(for sessionID: String) throws -> URL? { + let canonicalID = CodexLineageDiscovery.canonicalSessionID(sessionID) + if let known = self.filesByID[canonicalID] { + return known + } + if !self.didIndexRoots { + try self.indexRoots() + } + return self.filesByID[canonicalID] + } + + private mutating func indexRoots() throws { + self.didIndexRoots = true + var files: [URL] = [] + for root in self.roots { + try self.checkCancellation?() + guard let enumerator = FileManager.default.enumerator( + at: root, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles, .skipsPackageDescendants]) + else { continue } + while let fileURL = enumerator.nextObject() as? URL { + try self.checkCancellation?() + guard fileURL.pathExtension.lowercased() == "jsonl" else { continue } + files.append(fileURL) + } + } + + for fileURL in files.sorted(by: { $0.path < $1.path }) { + try self.checkCancellation?() + if let ownerID = CostUsageScanner.codexRolloutOwnerID(fileURL: fileURL) { + let canonicalID = CodexLineageDiscovery.canonicalSessionID(ownerID) + self.filesByID[canonicalID] = self.filesByID[canonicalID] ?? fileURL + } + if let metadataID = try CostUsageScanner.parseCodexSessionIdentifier( + fileURL: fileURL, + checkCancellation: self.checkCancellation) + { + let canonicalID = CodexLineageDiscovery.canonicalSessionID(metadataID) + self.filesByID[canonicalID] = self.filesByID[canonicalID] ?? fileURL + } + } + } + } +} diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 9bda419158..b52b9e4bac 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -1829,7 +1829,7 @@ enum CostUsageScanner { CodexLineageLedger.Totals(input: totals.input, cached: totals.cached, output: totals.output) } - private static func codexRolloutOwnerID(fileURL: URL) -> String? { + static func codexRolloutOwnerID(fileURL: URL) -> String? { let stem = fileURL.deletingPathExtension().lastPathComponent guard stem.count >= 36 else { return nil } let candidate = String(stem.suffix(36)) diff --git a/Tests/CodexBarTests/CodexLineageDiscoveryTests.swift b/Tests/CodexBarTests/CodexLineageDiscoveryTests.swift new file mode 100644 index 0000000000..9ab3126144 --- /dev/null +++ b/Tests/CodexBarTests/CodexLineageDiscoveryTests.swift @@ -0,0 +1,120 @@ +import Foundation +import Testing +@testable import CodexBarCore + +struct CodexLineageDiscoveryTests { + @Test + func `referenced parents cross the normal window and active archive roots transitively`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let grandparentID = "11111111-1111-4111-8111-111111111111" + let parentID = "22222222-2222-4222-8222-222222222222" + let childID = "33333333-3333-4333-8333-333333333333" + let unrelatedID = "44444444-4444-4444-8444-444444444444" + let grandparent = try Self.writeRollout( + root: environment.codexArchivedSessionsRoot, + relativeDirectory: "", + ownerID: grandparentID, + metadataID: grandparentID) + let parent = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2025/01/01", + ownerID: parentID, + metadataID: parentID, + parentID: grandparentID) + let child = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2026/07/09", + ownerID: childID, + metadataID: childID, + parentID: parentID) + _ = try Self.writeRollout( + root: environment.codexArchivedSessionsRoot, + relativeDirectory: "", + ownerID: unrelatedID, + metadataID: unrelatedID) + + let report = try CodexLineageDiscovery.discover( + includedFiles: [child], + roots: [environment.codexSessionsRoot, environment.codexArchivedSessionsRoot]) + + #expect(Set(report.documents.map(\.ownerID)) == [childID, parentID, grandparentID]) + #expect(report.documents.map(\.ownerID).contains(unrelatedID) == false) + #expect(report.referencedParentDocumentCount == 2) + #expect(report.unresolvedParentIDs.isEmpty) + #expect(FileManager.default.fileExists(atPath: parent.path)) + #expect(FileManager.default.fileExists(atPath: grandparent.path)) + } + + @Test + func `missing referenced parents are diagnosed without widening included documents`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let child = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2026/07/09", + ownerID: "55555555-5555-4555-8555-555555555555", + metadataID: "55555555-5555-4555-8555-555555555555", + parentID: "66666666-6666-4666-8666-666666666666") + + let report = try CodexLineageDiscovery.discover( + includedFiles: [child], + roots: [environment.codexSessionsRoot, environment.codexArchivedSessionsRoot]) + + #expect(report.documents.count == 1) + #expect(report.referencedParentDocumentCount == 0) + #expect(report.unresolvedParentIDs == ["66666666-6666-4666-8666-666666666666"]) + } + + @Test + func `uuid parent references are matched case insensitively against filename owners`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let parentID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + _ = try Self.writeRollout( + root: environment.codexArchivedSessionsRoot, + relativeDirectory: "", + ownerID: parentID, + metadataID: "parent-metadata-alias") + let child = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2026/07/09", + ownerID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + metadataID: "child-metadata", + parentID: parentID.uppercased()) + + let report = try CodexLineageDiscovery.discover( + includedFiles: [child], + roots: [environment.codexSessionsRoot, environment.codexArchivedSessionsRoot]) + + #expect(report.documents.map(\.ownerID).contains(parentID)) + #expect(report.referencedParentDocumentCount == 1) + #expect(report.unresolvedParentIDs.isEmpty) + } + + private static func writeRollout( + root: URL, + relativeDirectory: String, + ownerID: String, + metadataID: String, + parentID: String? = nil) throws -> URL + { + let directory = relativeDirectory.isEmpty + ? root + : root.appendingPathComponent(relativeDirectory, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let fileURL = directory.appendingPathComponent( + "rollout-2025-01-01T00-00-00-\(ownerID).jsonl") + var metadata = #"{"type":"session_meta","payload":{"id":"\#(metadataID)""# + if let parentID { + metadata += #", "forked_from_id":"\#(parentID)""# + } + metadata += "}}\n" + metadata += #"{"type":"event_msg","timestamp":"2026-07-09T12:00:00Z","payload":{"# + metadata += #""type":"token_count","info":{"last_token_usage":{"input_tokens":10,"# + metadata += #""cached_input_tokens":0,"output_tokens":1},"total_token_usage":{"input_tokens":10,"# + metadata += #""cached_input_tokens":0,"output_tokens":1}}}}"# + try metadata.write(to: fileURL, atomically: true, encoding: .utf8) + return fileURL + } +}