Feature/Fix: Add a shadow lineage ledger for Codex token accounting

Sources/CodexBarCore/Generated/CodexParserHash.generated.swift

@@ -1,5 +1,5 @@
 // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand.
 
 enum CodexParserHash {
-    static let value = "cdef6eb9658a43e2"
+    static let value = "d820fd60567bfc57"
 }

Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift

@@ -0,0 +1,192 @@
+import Foundation
+
+/// Experimental accounting model for Codex rollout families.
+///
+/// Rollout files are overlapping physical views of a logical lineage. The ledger builds the
+/// transitive lineage first, then admits each complete token observation once per lineage.
+/// It intentionally does not participate in production cost totals yet.
+enum CodexLineageLedger {
+    struct Totals: Equatable, Hashable, Sendable {
+        var input: Int
+        var cached: Int
+        var output: Int
+
+        static let zero = Self(input: 0, cached: 0, output: 0)
+
+        mutating func add(_ other: Self) {
+            self.input += other.input
+            self.cached += other.cached
+            self.output += other.output
+        }
+    }
+
+    struct Observation: Equatable, Sendable {
+        let eventID: String?
+        let timestamp: String
+        let last: Totals
+        let total: Totals
+
+        init(eventID: String? = nil, timestamp: String, last: Totals, total: Totals) {
+            self.eventID = eventID
+            self.timestamp = timestamp
+            self.last = last
+            self.total = total
+        }
+    }
+
+    struct Document: Equatable, Sendable {
+        /// Canonical owner from the rollout filename when available.
+        let ownerID: String
+        /// Session identity persisted in metadata. Fork copies may retain an ancestor identity.
+        let metadataSessionID: String?
+        let parentSessionID: String?
+        let observations: [Observation]
+    }
+
+    struct Report: Equatable, Sendable {
+        let utcDays: [String: Totals]
+        let localDays: [String: Totals]
+        let componentCount: Int
+        let acceptedObservationCount: Int
+        let duplicateObservationCount: Int
+    }
+
+    enum LedgerError: Error, Equatable {
+        case emptyOwnerID
+        case invalidTimestamp(String)
+    }
+
+    static func reconcile(documents: [Document], localTimeZone: TimeZone) throws -> Report {
+        var graph = DisjointSet()
+        for document in documents {
+            guard !document.ownerID.isEmpty else { throw LedgerError.emptyOwnerID }
+            graph.insert(document.ownerID)
+            if let metadataSessionID = Self.nonEmpty(document.metadataSessionID) {
+                graph.union(document.ownerID, metadataSessionID)
+            }
+            if let parentSessionID = Self.nonEmpty(document.parentSessionID) {
+                graph.union(document.ownerID, parentSessionID)
+            }
+        }
+
+        var acceptedByComponent: [String: [ObservationIdentity: AcceptedObservation]] = [:]
+        var acceptedFingerprintByComponentOwner: [String: [String: [Fingerprint: ObservationIdentity]]] = [:]
+        var physicalObservationCount = 0
+        for document in documents {
+            let componentID = graph.find(document.ownerID)
+            var accepted = acceptedByComponent[componentID] ?? [:]
+            var acceptedFingerprintByOwner = acceptedFingerprintByComponentOwner[componentID] ?? [:]
+            for observation in document.observations {
+                physicalObservationCount += 1
+                let date = try Self.date(from: observation.timestamp)
+                let fingerprint = Fingerprint(last: observation.last, total: observation.total)
+                let proposedIdentity = ObservationIdentity(
+                    eventID: Self.nonEmpty(observation.eventID),
+                    fingerprint: fingerprint)
+                let identity = accepted[proposedIdentity] == nil
+                    ? acceptedFingerprintByOwner[document.ownerID]?[fingerprint] ?? proposedIdentity
+                    : proposedIdentity
+                if let existing = accepted[identity], existing.date <= date {
+                    continue
+                }
+                accepted[identity] = AcceptedObservation(date: date, last: observation.last)
+                if identity == proposedIdentity {
+                    acceptedFingerprintByOwner[document.ownerID, default: [:]][fingerprint] = identity
+                }
+            }
+            acceptedByComponent[componentID] = accepted
+            acceptedFingerprintByComponentOwner[componentID] = acceptedFingerprintByOwner
+        }
+
+        var utcDays: [String: Totals] = [:]
+        var localDays: [String: Totals] = [:]
+        var acceptedObservationCount = 0
+        for accepted in acceptedByComponent.values {
+            acceptedObservationCount += accepted.count
+            for observation in accepted.values {
+                Self.add(observation.last, on: observation.date, timeZone: .gmt, to: &utcDays)
+                Self.add(observation.last, on: observation.date, timeZone: localTimeZone, to: &localDays)
+            }
+        }
+
+        return Report(
+            utcDays: utcDays,
+            localDays: localDays,
+            componentCount: Set(documents.map { graph.find($0.ownerID) }).count,
+            acceptedObservationCount: acceptedObservationCount,
+            duplicateObservationCount: physicalObservationCount - acceptedObservationCount)
+    }
+
+    private struct Fingerprint: Equatable, Hashable {
+        let last: Totals
+        let total: Totals
+    }
+
+    private enum ObservationIdentity: Equatable, Hashable {
+        case event(String, Fingerprint)
+        case fingerprint(Fingerprint)
+
+        init(eventID: String?, fingerprint: Fingerprint) {
+            self = eventID.map { .event($0, fingerprint) } ?? .fingerprint(fingerprint)
+        }
+    }
+
+    private struct AcceptedObservation {
+        let date: Date
+        let last: Totals
+    }
+
+    private static func nonEmpty(_ value: String?) -> String? {
+        guard let value, !value.isEmpty else { return nil }
+        return value
+    }
+
+    private static func date(from timestamp: String) throws -> Date {
+        guard let date = CostUsageScanner.dateFromTimestamp(timestamp) else {
+            throw LedgerError.invalidTimestamp(timestamp)
+        }
+        return date
+    }
+
+    private static func add(
+        _ totals: Totals,
+        on date: Date,
+        timeZone: TimeZone,
+        to days: inout [String: Totals])
+    {
+        var calendar = Calendar(identifier: .gregorian)
+        calendar.timeZone = timeZone
+        let components = calendar.dateComponents([.year, .month, .day], from: date)
+        guard let year = components.year, let month = components.month, let day = components.day else { return }
+        let key = String(format: "%04d-%02d-%02d", year, month, day)
+        var dayTotals = days[key] ?? .zero
+        dayTotals.add(totals)
+        days[key] = dayTotals
+    }
+
+    private struct DisjointSet {
+        private var parents: [String: String] = [:]
+
+        mutating func insert(_ item: String) {
+            if self.parents[item] == nil {
+                self.parents[item] = item
+            }
+        }
+
+        mutating func find(_ item: String) -> String {
+            self.insert(item)
+            guard let parent = self.parents[item], parent != item else { return item }
+            let root = self.find(parent)
+            self.parents[item] = root
+            return root
+        }
+
+        mutating func union(_ first: String, _ second: String) {
+            let firstRoot = self.find(first)
+            let secondRoot = self.find(second)
+            if firstRoot != secondRoot {
+                self.parents[secondRoot] = firstRoot
+            }
+        }
+    }
+}

Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift

@@ -98,6 +98,13 @@ enum CostUsageScanner {
         let totals: CostUsageCodexTotals
     }
 
+    private struct CodexParsedTokenEvidence {
+        let sessionId: String?
+        let forkedFromId: String?
+        let snapshots: [CodexTimestampedTotals]
+        let observations: [CodexLineageLedger.Observation]
+    }
+
     enum CodexForkBaseline {
         case resolved(CostUsageCodexTotals?)
         case unresolved
@@ -1680,14 +1687,17 @@ enum CostUsageScanner {
 
     private static func parseCodexTokenSnapshots(
         fileURL: URL,
-        checkCancellation: CancellationCheck? = nil) throws -> (
-        sessionId: String?,
-        snapshots: [CodexTimestampedTotals])
+        collectLineageObservations: Bool = false,
+        checkCancellation: CancellationCheck? = nil) throws -> CodexParsedTokenEvidence
     {
         var sessionId: String?
+        var forkedFromId: String?
         var accumulator = CodexSnapshotAccumulator()
         var snapshots: [CodexTimestampedTotals] = []
+        var observations: [CodexLineageLedger.Observation] = []
         var warnedAboutUnparsedTimestamp = false
+        var currentTurnID: String?
+        var tokenEventCountByTurn: [String: Int] = [:]
 
         func parsedSnapshotDate(timestamp: String) -> Date? {
             let date = Self.dateFromTimestamp(timestamp)
@@ -1701,13 +1711,31 @@ enum CostUsageScanner {
             return date
         }
 
-        func appendSnapshot(timestamp: String, last: CostUsageCodexTotals?, total: CostUsageCodexTotals?) {
+        func appendSnapshot(
+            timestamp: String,
+            turnID: String?,
+            last: CostUsageCodexTotals?,
+            total: CostUsageCodexTotals?)
+        {
             guard last != nil || total != nil else { return }
             let counted = accumulator.apply(last: last, total: total)
             snapshots.append(CodexTimestampedTotals(
                 timestamp: timestamp,
                 date: parsedSnapshotDate(timestamp: timestamp),
                 totals: counted))
+            guard collectLineageObservations else { return }
+            let eventID = turnID.map { turnID in
+                let ordinal = tokenEventCountByTurn[turnID, default: 0]
+                tokenEventCountByTurn[turnID] = ordinal + 1
+                return "\(turnID):\(ordinal)"
+            }
+            if let last, let total {
+                observations.append(CodexLineageLedger.Observation(
+                    eventID: eventID,
+                    timestamp: timestamp,
+                    last: Self.lineageTotals(last),
+                    total: Self.lineageTotals(total)))
+            }
         }
 
         do {
@@ -1724,9 +1752,18 @@ enum CostUsageScanner {
                             if sessionId == nil {
                                 sessionId = metadata.sessionId
                             }
+                            if forkedFromId == nil {
+                                forkedFromId = metadata.forkedFromId
+                            }
                         case let .tokenCount(record):
-                            appendSnapshot(timestamp: record.timestamp, last: record.last, total: record.total)
-                        case .turnContext, .taskStarted:
+                            appendSnapshot(
+                                timestamp: record.timestamp,
+                                turnID: record.turnID ?? currentTurnID,
+                                last: record.last,
+                                total: record.total)
+                        case let .taskStarted(turnID):
+                            currentTurnID = turnID
+                        case .turnContext:
                             break
                         }
                         return
@@ -1746,11 +1783,18 @@ enum CostUsageScanner {
                                     ?? obj["sessionId"] as? String
                                     ?? obj["id"] as? String
                             }
+                            if forkedFromId == nil {
+                                forkedFromId = Self.codexForkParentId(from: payload)
+                            }
                             return
                         }
 
                         guard obj["type"] as? String == "event_msg" else { return }
                         guard let payload = obj["payload"] as? [String: Any] else { return }
+                        if payload["type"] as? String == "task_started" {
+                            currentTurnID = Self.codexTurnID(from: payload)
+                            return
+                        }
                         guard payload["type"] as? String == "token_count" else { return }
                         guard let info = payload["info"] as? [String: Any] else { return }
                         guard let timestamp = obj["timestamp"] as? String else { return }
@@ -1774,7 +1818,11 @@ enum CostUsageScanner {
                                 cached: max(0, toInt($0["cached_input_tokens"] ?? $0["cache_read_input_tokens"])),
                                 output: max(0, toInt($0["output_tokens"])))
                         }
-                        appendSnapshot(timestamp: timestamp, last: last, total: total)
+                        appendSnapshot(
+                            timestamp: timestamp,
+                            turnID: Self.codexTurnID(from: payload) ?? currentTurnID,
+                            last: last,
+                            total: total)
                     }
                 })
         } catch is CancellationError {
@@ -1785,7 +1833,47 @@ enum CostUsageScanner {
                 metadata: ["path": fileURL.path, "error": error.localizedDescription])
         }
 
-        return (sessionId, snapshots)
+        return CodexParsedTokenEvidence(
+            sessionId: sessionId,
+            forkedFromId: forkedFromId,
+            snapshots: snapshots,
+            observations: observations)
+    }
+
+    static func parseCodexLineageDocument(
+        fileURL: URL,
+        checkCancellation: CancellationCheck? = nil) throws -> CodexLineageLedger.Document
+    {
+        let parsed = try Self.parseCodexTokenSnapshots(
+            fileURL: fileURL,
+            collectLineageObservations: true,
+            checkCancellation: checkCancellation)
+        return CodexLineageLedger.Document(
+            ownerID: Self.codexRolloutOwnerID(fileURL: fileURL) ?? parsed.sessionId ?? fileURL.standardizedFileURL.path,
+            metadataSessionID: parsed.sessionId,
+            parentSessionID: parsed.forkedFromId,
+            observations: parsed.observations)
+    }
+
+    static func parseCodexTokenEvidenceCountsForTesting(
+        fileURL: URL,
+        collectLineageObservations: Bool) throws -> (snapshots: Int, observations: Int)
+    {
+        let parsed = try Self.parseCodexTokenSnapshots(
+            fileURL: fileURL,
+            collectLineageObservations: collectLineageObservations)
+        return (parsed.snapshots.count, parsed.observations.count)
+    }
+
+    private static func lineageTotals(_ totals: CostUsageCodexTotals) -> CodexLineageLedger.Totals {
+        CodexLineageLedger.Totals(input: totals.input, cached: totals.cached, output: totals.output)
+    }
+
+    private static func codexRolloutOwnerID(fileURL: URL) -> String? {
+        let stem = fileURL.deletingPathExtension().lastPathComponent
+        guard stem.count >= 36 else { return nil }
+        let candidate = String(stem.suffix(36))
+        return UUID(uuidString: candidate) == nil ? nil : candidate.lowercased()
     }
 
     static func parseCodexFile(