Repository navigation
Feature/Fix: Add a shadow lineage ledger for Codex token accounting #2155
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
iam-brain
wants to merge
7
commits into
steipete:main
from
iam-brain:iam-brain/lineage-ledger-foundation
Closed
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
9245047
Add shadow Codex lineage ledger
iam-brain c731bd1
Adapt rollout snapshots for lineage accounting
iam-brain 41dc8aa
Preserve rollout event identity
iam-brain 5cce73b
Refresh lineage parser fingerprint
iam-brain 2e80ecb
Format lineage identity test
iam-brain 8bf1f6e
Avoid unused lineage observations
iam-brain f917e1a
Harden lineage observation identity
iam-brain File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. | ||
|
|
||
| enum CodexParserHash { | ||
| static let value = "cdef6eb9658a43e2" | ||
| static let value = "d820fd60567bfc57" | ||
| } |
182 changes: 182 additions & 0 deletions
182
Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| import Foundation | ||
|
|
||
| /// Experimental accounting model for Codex rollout families. | ||
| /// | ||
| /// Rollout files are overlapping physical views of a logical lineage. The ledger builds the | ||
| /// transitive lineage first, then admits each complete token observation once per lineage. | ||
| /// It intentionally does not participate in production cost totals yet. | ||
| enum CodexLineageLedger { | ||
| struct Totals: Equatable, Hashable, Sendable { | ||
| var input: Int | ||
| var cached: Int | ||
| var output: Int | ||
|
|
||
| static let zero = Self(input: 0, cached: 0, output: 0) | ||
|
|
||
| mutating func add(_ other: Self) { | ||
| self.input += other.input | ||
| self.cached += other.cached | ||
| self.output += other.output | ||
| } | ||
| } | ||
|
|
||
| struct Observation: Equatable, Sendable { | ||
| let eventID: String? | ||
| let timestamp: String | ||
| let last: Totals | ||
| let total: Totals | ||
|
|
||
| init(eventID: String? = nil, timestamp: String, last: Totals, total: Totals) { | ||
| self.eventID = eventID | ||
| self.timestamp = timestamp | ||
| self.last = last | ||
| self.total = total | ||
| } | ||
| } | ||
|
|
||
| struct Document: Equatable, Sendable { | ||
| /// Canonical owner from the rollout filename when available. | ||
| let ownerID: String | ||
| /// Session identity persisted in metadata. Fork copies may retain an ancestor identity. | ||
| let metadataSessionID: String? | ||
| let parentSessionID: String? | ||
| let observations: [Observation] | ||
| } | ||
|
|
||
| struct Report: Equatable, Sendable { | ||
| let utcDays: [String: Totals] | ||
| let localDays: [String: Totals] | ||
| let componentCount: Int | ||
| let acceptedObservationCount: Int | ||
| let duplicateObservationCount: Int | ||
| } | ||
|
|
||
| enum LedgerError: Error, Equatable { | ||
| case emptyOwnerID | ||
| case invalidTimestamp(String) | ||
| } | ||
|
|
||
| static func reconcile(documents: [Document], localTimeZone: TimeZone) throws -> Report { | ||
| var graph = DisjointSet() | ||
| for document in documents { | ||
| guard !document.ownerID.isEmpty else { throw LedgerError.emptyOwnerID } | ||
| graph.insert(document.ownerID) | ||
| if let metadataSessionID = Self.nonEmpty(document.metadataSessionID) { | ||
| graph.union(document.ownerID, metadataSessionID) | ||
| } | ||
| if let parentSessionID = Self.nonEmpty(document.parentSessionID) { | ||
| graph.union(document.ownerID, parentSessionID) | ||
| } | ||
| } | ||
|
|
||
| var acceptedByComponent: [String: [ObservationIdentity: AcceptedObservation]] = [:] | ||
| var physicalObservationCount = 0 | ||
| for document in documents { | ||
| let componentID = graph.find(document.ownerID) | ||
| var accepted = acceptedByComponent[componentID] ?? [:] | ||
| for observation in document.observations { | ||
| physicalObservationCount += 1 | ||
| let date = try Self.date(from: observation.timestamp) | ||
| let identity = ObservationIdentity( | ||
| eventID: Self.nonEmpty(observation.eventID), | ||
| fingerprint: Fingerprint(last: observation.last, total: observation.total)) | ||
| if let existing = accepted[identity], existing.date <= date { | ||
| continue | ||
| } | ||
| accepted[identity] = AcceptedObservation(date: date, last: observation.last) | ||
| } | ||
| acceptedByComponent[componentID] = accepted | ||
| } | ||
|
|
||
| var utcDays: [String: Totals] = [:] | ||
| var localDays: [String: Totals] = [:] | ||
| var acceptedObservationCount = 0 | ||
| for accepted in acceptedByComponent.values { | ||
| acceptedObservationCount += accepted.count | ||
| for observation in accepted.values { | ||
| Self.add(observation.last, on: observation.date, timeZone: .gmt, to: &utcDays) | ||
| Self.add(observation.last, on: observation.date, timeZone: localTimeZone, to: &localDays) | ||
| } | ||
| } | ||
|
|
||
| return Report( | ||
| utcDays: utcDays, | ||
| localDays: localDays, | ||
| componentCount: Set(documents.map { graph.find($0.ownerID) }).count, | ||
| acceptedObservationCount: acceptedObservationCount, | ||
| duplicateObservationCount: physicalObservationCount - acceptedObservationCount) | ||
| } | ||
|
|
||
| private struct Fingerprint: Equatable, Hashable { | ||
| let last: Totals | ||
| let total: Totals | ||
| } | ||
|
|
||
| private enum ObservationIdentity: Equatable, Hashable { | ||
| case event(String) | ||
| case fingerprint(Fingerprint) | ||
|
|
||
| init(eventID: String?, fingerprint: Fingerprint) { | ||
| self = eventID.map(Self.event) ?? .fingerprint(fingerprint) | ||
| } | ||
| } | ||
|
|
||
| private struct AcceptedObservation { | ||
| let date: Date | ||
| let last: Totals | ||
| } | ||
|
|
||
| private static func nonEmpty(_ value: String?) -> String? { | ||
| guard let value, !value.isEmpty else { return nil } | ||
| return value | ||
| } | ||
|
|
||
| private static func date(from timestamp: String) throws -> Date { | ||
| guard let date = CostUsageScanner.dateFromTimestamp(timestamp) else { | ||
| throw LedgerError.invalidTimestamp(timestamp) | ||
| } | ||
| return date | ||
| } | ||
|
|
||
| private static func add( | ||
| _ totals: Totals, | ||
| on date: Date, | ||
| timeZone: TimeZone, | ||
| to days: inout [String: Totals]) | ||
| { | ||
| var calendar = Calendar(identifier: .gregorian) | ||
| calendar.timeZone = timeZone | ||
| let components = calendar.dateComponents([.year, .month, .day], from: date) | ||
| guard let year = components.year, let month = components.month, let day = components.day else { return } | ||
| let key = String(format: "%04d-%02d-%02d", year, month, day) | ||
| var dayTotals = days[key] ?? .zero | ||
| dayTotals.add(totals) | ||
| days[key] = dayTotals | ||
| } | ||
|
|
||
| private struct DisjointSet { | ||
| private var parents: [String: String] = [:] | ||
|
|
||
| mutating func insert(_ item: String) { | ||
| if self.parents[item] == nil { | ||
| self.parents[item] = item | ||
| } | ||
| } | ||
|
|
||
| mutating func find(_ item: String) -> String { | ||
| self.insert(item) | ||
| guard let parent = self.parents[item], parent != item else { return item } | ||
| let root = self.find(parent) | ||
| self.parents[item] = root | ||
| return root | ||
| } | ||
|
|
||
| mutating func union(_ first: String, _ second: String) { | ||
| let firstRoot = self.find(first) | ||
| let secondRoot = self.find(second) | ||
| if firstRoot != secondRoot { | ||
| self.parents[secondRoot] = firstRoot | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When parsed rollout rows carry a turn id,
parseCodexTokenSnapshotsassigns eachtoken_counta distinctturnID:ordinalevent ID. For an unchanged same-turn re-emission, the fingerprint is identical but the ordinal advances, and this initializer chooses.eventwhenevereventIDis non-nil, soreconcileadmits both rows and double-counts theirlasttokens instead of collapsing the repeated state.Useful? React with 👍 / 👎.