From a140e7df7e8850502703ba8a03545d0b0d253be5 Mon Sep 17 00:00:00 2001 From: Bryan Font Date: Mon, 13 Jul 2026 13:59:06 -0400 Subject: [PATCH] Classify Codex lineage residuals --- .../CodexLineageResidualClassifier.swift | 189 ++++++++++++++++++ .../CodexLineageResidualClassifierTests.swift | 147 ++++++++++++++ 2 files changed, 336 insertions(+) create mode 100644 Sources/CodexBarCore/Providers/Codex/CodexLineageResidualClassifier.swift create mode 100644 Tests/CodexBarTests/CodexLineageResidualClassifierTests.swift diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageResidualClassifier.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageResidualClassifier.swift new file mode 100644 index 0000000000..5311942c77 --- /dev/null +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageResidualClassifier.swift @@ -0,0 +1,189 @@ +import Foundation + +/// Classifies differences between local Codex accounting paths and a finalized UTC usage source. +/// +/// This is an analysis seam, not an oracle-tuning mechanism. Callers supply independently observed +/// totals and corpus diagnostics; the classifier only applies a documented, bounded policy. +enum CodexLineageResidualClassifier { + enum Classification: String, Equatable, Sendable { + case invalidInput + case provisional + case withinTolerance + case unavailableHistory + case unsupportedEventShape + case utcLocalAttribution + case accountingSemantics + case containment + case ledgerDefect + } + + struct Evidence: Equatable, Sendable { + var localCorpusWasExhaustive = false + var rejectedObservationCount = 0 + var unresolvedParentCount = 0 + var duplicateObservationCount = 0 + } + + struct Sample: Equatable, Sendable { + let day: String + let referenceTokens: Int + let isReferenceFinalized: Bool + let isOrdinaryDay: Bool + let legacyTokens: Int + let ledgerUTCTokens: Int + let ledgerLocalTokens: Int + let evidence: Evidence + } + + struct DayResult: Equatable, Sendable { + let day: String + let classification: Classification + let legacyAbsoluteError: Int? + let ledgerAbsoluteError: Int? + } + + struct Report: Equatable, Sendable { + let days: [DayResult] + let finalizedReferenceTokens: Int + let finalizedLegacyTokens: Int + let finalizedLedgerTokens: Int + let legacyAbsoluteError: Int + let ledgerAbsoluteError: Int + let ordinaryDayRegressionCount: Int + let invalidSampleCount: Int + + var improvesAggregateError: Bool { + self.ledgerAbsoluteError < self.legacyAbsoluteError + } + } + + struct Policy: Equatable, Sendable { + /// A residual no larger than this fraction of the reference is not assigned a speculative cause. + let largeResidualFraction: Double + /// Ordinary days may move by this fraction before the ledger is considered regressive. + let ordinaryDayRegressionFraction: Double + + static let validation = Self( + largeResidualFraction: 0.05, + ordinaryDayRegressionFraction: 0.01) + } + + static func classify(samples: [Sample], policy: Policy = .validation) -> Report { + let ordered = samples.sorted { $0.day < $1.day } + let days = ordered.map { sample -> DayResult in + guard Self.isValid(sample: sample, policy: policy) else { + return DayResult( + day: sample.day, + classification: .invalidInput, + legacyAbsoluteError: nil, + ledgerAbsoluteError: nil) + } + guard sample.isReferenceFinalized else { + return DayResult( + day: sample.day, + classification: .provisional, + legacyAbsoluteError: nil, + ledgerAbsoluteError: nil) + } + let legacyError = Self.absoluteDifference(sample.legacyTokens, sample.referenceTokens) + let ledgerError = Self.absoluteDifference(sample.ledgerUTCTokens, sample.referenceTokens) + return DayResult( + day: sample.day, + classification: Self.classification( + sample: sample, + legacyError: legacyError, + ledgerError: ledgerError, + policy: policy), + legacyAbsoluteError: legacyError, + ledgerAbsoluteError: ledgerError) + } + + let finalized = ordered.filter { $0.isReferenceFinalized && Self.isValid(sample: $0, policy: policy) } + let referenceTokens = finalized.reduce(0) { Self.saturatingSum($0, $1.referenceTokens) } + let legacyTokens = finalized.reduce(0) { Self.saturatingSum($0, $1.legacyTokens) } + let ledgerTokens = finalized.reduce(0) { Self.saturatingSum($0, $1.ledgerUTCTokens) } + let ordinaryDayRegressionCount = finalized.count { sample in + guard sample.isOrdinaryDay else { return false } + let allowed = Self.threshold( + reference: sample.referenceTokens, + fraction: policy.ordinaryDayRegressionFraction) + return Self.absoluteDifference(sample.ledgerUTCTokens, sample.referenceTokens) + > Self.saturatingSum(Self.absoluteDifference(sample.legacyTokens, sample.referenceTokens), allowed) + } + + return Report( + days: days, + finalizedReferenceTokens: referenceTokens, + finalizedLegacyTokens: legacyTokens, + finalizedLedgerTokens: ledgerTokens, + legacyAbsoluteError: Self.absoluteDifference(legacyTokens, referenceTokens), + ledgerAbsoluteError: Self.absoluteDifference(ledgerTokens, referenceTokens), + ordinaryDayRegressionCount: ordinaryDayRegressionCount, + invalidSampleCount: days.count { $0.classification == .invalidInput }) + } + + private static func classification( + sample: Sample, + legacyError: Int, + ledgerError: Int, + policy: Policy) -> Classification + { + let largeResidual = Self.threshold( + reference: sample.referenceTokens, + fraction: policy.largeResidualFraction) + guard ledgerError > largeResidual else { return .withinTolerance } + if sample.evidence.rejectedObservationCount > 0 { + return .unsupportedEventShape + } + + let ordinaryAllowance = Self.threshold( + reference: sample.referenceTokens, + fraction: policy.ordinaryDayRegressionFraction) + if sample.isOrdinaryDay, ledgerError > Self.saturatingSum(legacyError, ordinaryAllowance) { + return .ledgerDefect + } + + let localError = Self.absoluteDifference(sample.ledgerLocalTokens, sample.referenceTokens) + if Self.saturatingSum(ledgerError, ordinaryAllowance) < localError { + return .utcLocalAttribution + } + if sample.evidence.unresolvedParentCount > 0 { + return .containment + } + if sample.evidence.localCorpusWasExhaustive, sample.ledgerUTCTokens < sample.referenceTokens { + return .unavailableHistory + } + if sample.evidence.duplicateObservationCount > 0, ledgerError < legacyError { + return .containment + } + return .accountingSemantics + } + + private static func threshold(reference: Int, fraction: Double) -> Int { + let value = (Double(reference) * fraction).rounded(.up) + return value >= Double(Int.max) ? Int.max : Int(value) + } + + private static func absoluteDifference(_ lhs: Int, _ rhs: Int) -> Int { + lhs >= rhs ? lhs - rhs : rhs - lhs + } + + private static func isValid(sample: Sample, policy: Policy) -> Bool { + sample.referenceTokens >= 0 && + sample.legacyTokens >= 0 && + sample.ledgerUTCTokens >= 0 && + sample.ledgerLocalTokens >= 0 && + sample.evidence.rejectedObservationCount >= 0 && + sample.evidence.unresolvedParentCount >= 0 && + sample.evidence.duplicateObservationCount >= 0 && + policy.largeResidualFraction.isFinite && + policy.largeResidualFraction >= 0 && + policy.ordinaryDayRegressionFraction.isFinite && + policy.ordinaryDayRegressionFraction >= 0 + } + + private static func saturatingSum(_ lhs: Int, _ rhs: Int) -> Int { + let (sum, overflow) = lhs.addingReportingOverflow(rhs) + return overflow ? Int.max : sum + } +} diff --git a/Tests/CodexBarTests/CodexLineageResidualClassifierTests.swift b/Tests/CodexBarTests/CodexLineageResidualClassifierTests.swift new file mode 100644 index 0000000000..b583256e00 --- /dev/null +++ b/Tests/CodexBarTests/CodexLineageResidualClassifierTests.swift @@ -0,0 +1,147 @@ +import Testing +@testable import CodexBarCore + +struct CodexLineageResidualClassifierTests { + @Test + func `sanitized finalized UTC replay materially closes the aggregate gap`() { + let report = CodexLineageResidualClassifier.classify(samples: Self.forkHeavySamples) + + #expect(report.finalizedReferenceTokens == 3_692_873_480) + #expect(report.finalizedLegacyTokens == 2_882_545_128) + #expect(report.finalizedLedgerTokens == 3_589_444_942) + #expect(report.legacyAbsoluteError == 810_328_352) + #expect(report.ledgerAbsoluteError == 103_428_538) + #expect(report.improvesAggregateError) + #expect(report.days.allSatisfy { day in + guard let legacy = day.legacyAbsoluteError, let ledger = day.ledgerAbsoluteError else { return true } + return ledger < legacy + }) + } + + @Test + func `large undercount is unavailable history only after exhaustive local checks`() { + let verified = CodexLineageResidualClassifier.classify(samples: [Self.forkHeavySamples[0]]) + #expect(verified.days.first?.classification == .unavailableHistory) + + var incomplete = Self.forkHeavySamples[0] + incomplete = .init( + day: incomplete.day, + referenceTokens: incomplete.referenceTokens, + isReferenceFinalized: true, + isOrdinaryDay: false, + legacyTokens: incomplete.legacyTokens, + ledgerUTCTokens: incomplete.ledgerUTCTokens, + ledgerLocalTokens: incomplete.ledgerLocalTokens, + evidence: .init(localCorpusWasExhaustive: false)) + let unverified = CodexLineageResidualClassifier.classify(samples: [incomplete]) + #expect(unverified.days.first?.classification == .accountingSemantics) + } + + @Test + func `ordinary non-fork days do not materially regress`() { + let samples = [ + Self.ordinary(day: "2026-06-29", legacy: 120_000_000, ledger: 120_000_000), + Self.ordinary(day: "2026-06-30", legacy: 180_000_000, ledger: 180_000_000), + Self.ordinary(day: "2026-07-05", legacy: 290_018_710, ledger: 290_777_623), + Self.ordinary(day: "2026-07-06", legacy: 435_123_419, ledger: 435_123_419), + Self.ordinary(day: "2026-07-07", legacy: 324_480_000, ledger: 324_480_000), + Self.ordinary(day: "2026-07-08", legacy: 240_000_000, ledger: 240_000_000), + ] + + let report = CodexLineageResidualClassifier.classify(samples: samples) + #expect(report.ordinaryDayRegressionCount == 0) + #expect(report.days.allSatisfy { $0.classification == .withinTolerance }) + } + + @Test + func `reference day remains provisional until finalized`() { + let sample = CodexLineageResidualClassifier.Sample( + day: "2026-07-12", + referenceTokens: 618_121_840, + isReferenceFinalized: false, + isOrdinaryDay: false, + legacyTokens: 600_000_000, + ledgerUTCTokens: 615_000_000, + ledgerLocalTokens: 610_000_000, + evidence: .init()) + + let report = CodexLineageResidualClassifier.classify(samples: [sample]) + #expect(report.days.first?.classification == .provisional) + #expect(report.finalizedReferenceTokens == 0) + #expect(report.days.first?.ledgerAbsoluteError == nil) + } + + @Test + func `invalid token evidence is excluded without overflowing aggregate totals`() { + let invalid = CodexLineageResidualClassifier.Sample( + day: "2026-07-08", + referenceTokens: -1, + isReferenceFinalized: true, + isOrdinaryDay: true, + legacyTokens: 10, + ledgerUTCTokens: 10, + ledgerLocalTokens: 10, + evidence: .init()) + let large = CodexLineageResidualClassifier.Sample( + day: "2026-07-09", + referenceTokens: Int.max, + isReferenceFinalized: true, + isOrdinaryDay: false, + legacyTokens: Int.max, + ledgerUTCTokens: Int.max, + ledgerLocalTokens: Int.max, + evidence: .init()) + + let report = CodexLineageResidualClassifier.classify(samples: [invalid, large, large]) + + #expect(report.days.first?.classification == .invalidInput) + #expect(report.invalidSampleCount == 1) + #expect(report.finalizedReferenceTokens == Int.max) + #expect(report.finalizedLegacyTokens == Int.max) + #expect(report.finalizedLedgerTokens == Int.max) + #expect(report.legacyAbsoluteError == 0) + #expect(report.ledgerAbsoluteError == 0) + } + + private static let forkHeavySamples = [ + CodexLineageResidualClassifier.Sample( + day: "2026-07-09", + referenceTokens: 852_682_935, + isReferenceFinalized: true, + isOrdinaryDay: false, + legacyTokens: 946_818_053, + ledgerUTCTokens: 764_026_920, + ledgerLocalTokens: 764_026_920, + evidence: .init(localCorpusWasExhaustive: true, duplicateObservationCount: 100)), + CodexLineageResidualClassifier.Sample( + day: "2026-07-10", + referenceTokens: 1_580_199_588, + isReferenceFinalized: true, + isOrdinaryDay: false, + legacyTokens: 1_414_122_342, + ledgerUTCTokens: 1_510_013_760, + ledgerLocalTokens: 1_430_000_000, + evidence: .init(localCorpusWasExhaustive: true, duplicateObservationCount: 100)), + CodexLineageResidualClassifier.Sample( + day: "2026-07-11", + referenceTokens: 1_259_990_957, + isReferenceFinalized: true, + isOrdinaryDay: false, + legacyTokens: 521_604_733, + ledgerUTCTokens: 1_315_404_262, + ledgerLocalTokens: 1_200_000_000, + evidence: .init(localCorpusWasExhaustive: true, duplicateObservationCount: 100)), + ] + + private static func ordinary(day: String, legacy: Int, ledger: Int) -> CodexLineageResidualClassifier.Sample { + .init( + day: day, + referenceTokens: legacy, + isReferenceFinalized: true, + isOrdinaryDay: true, + legacyTokens: legacy, + ledgerUTCTokens: ledger, + ledgerLocalTokens: ledger, + evidence: .init(localCorpusWasExhaustive: true)) + } +}