diff --git a/CHANGELOG.md b/CHANGELOG.md index f2de4d7f8d..a457f9eed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Menu: the compact multi-account layout now covers every stacked multi-account list — token accounts on any provider and Codex accounts (flat lists; workspace-grouped Codex lists keep their sections). ### Fixed +- Codex: persist and budget fork-parent discovery so missing parents quiesce between inventory changes instead of sweeping every rollout on each refresh (#2525, #2538). Thanks @xx205, and @Helmi and @kiranmagic7 for the investigation! - Claude: Auto cold boot with Keychain disabled loads without manual refresh (#2494, fixes #2493). Thanks @gmkbenjamin! - Menu: no more stray floating "Refresh" tooltip beside the menu when switching tabs with the cursor over the actions area. - Providers: write the Factory and Cursor session files (bearer/refresh tokens, auth cookies) owner-only (0600), matching the codex/kimi/antigravity credential stores. diff --git a/Sources/CodexBar/CodexCostCatchUpPolicy.swift b/Sources/CodexBar/CodexCostCatchUpPolicy.swift new file mode 100644 index 0000000000..933d5efc92 --- /dev/null +++ b/Sources/CodexBar/CodexCostCatchUpPolicy.swift @@ -0,0 +1,116 @@ +import Foundation +import IOKit.ps + +enum CodexCostCatchUpMode: String, Sendable { + case automatic + case accelerated +} + +enum CodexCostCatchUpPowerSource: String, Sendable { + case ac + case battery + case unknown + + static func current() -> Self { + guard let info = IOPSCopyPowerSourcesInfo()?.takeRetainedValue(), + let source = IOPSGetProvidingPowerSourceType(info)?.takeUnretainedValue() as String? + else { + return .unknown + } + if source == kIOPSACPowerValue as String { + return .ac + } + if source == kIOPSBatteryPowerValue as String { + return .battery + } + return .unknown + } +} + +enum CodexCostCatchUpPauseReason: Sendable, Equatable { + case lowPower + case thermal + case user + case noProgress + case error(String) +} + +struct CodexCostCatchUpActivity: Sendable, Equatable { + enum Phase: Sendable, Equatable { + case indexing + case paused + case complete + } + + let phase: Phase + let mode: CodexCostCatchUpMode + let processedBytes: Int64 + let totalBytes: Int64 + let completedFiles: Int + let totalFiles: Int + let pauseReason: CodexCostCatchUpPauseReason? + let staleSnapshotUpdatedAt: Date? + + var fractionCompleted: Double? { + guard self.totalBytes > 0 else { + guard self.totalFiles > 0 else { return nil } + return min(1, max(0, Double(self.completedFiles) / Double(self.totalFiles))) + } + return min(1, max(0, Double(self.processedBytes) / Double(self.totalBytes))) + } +} + +struct CodexCostCatchUpPolicy: Sendable { + struct Input: Sendable { + let mode: CodexCostCatchUpMode + let previousActiveDuration: TimeInterval? + let powerSource: CodexCostCatchUpPowerSource + let lowPowerModeEnabled: Bool + let thermalState: ProcessInfo.ThermalState + } + + struct Decision: Sendable, Equatable { + enum Action: Sendable, Equatable { + case runAfter(TimeInterval) + case pause(TimeInterval, CodexCostCatchUpPauseReason) + } + + let action: Action + let targetDutyCycle: Double? + } + + static let automaticBurstDuration: TimeInterval = 2 + static let constrainedRetryDelay: TimeInterval = 60 + + func decision(for input: Input) -> Decision { + if input.thermalState == .critical { + return Decision( + action: .pause(Self.constrainedRetryDelay, .thermal), + targetDutyCycle: nil) + } + if input.mode == .automatic { + if input.lowPowerModeEnabled { + return Decision( + action: .pause(Self.constrainedRetryDelay, .lowPower), + targetDutyCycle: nil) + } + if input.thermalState == .serious { + return Decision( + action: .pause(Self.constrainedRetryDelay, .thermal), + targetDutyCycle: nil) + } + } + if input.mode == .accelerated { + return Decision(action: .runAfter(0), targetDutyCycle: 1) + } + + let dutyCycle = switch input.powerSource { + case .ac: 0.20 + case .battery: 0.05 + case .unknown: 0.15 + } + let activeDuration = max(0, input.previousActiveDuration ?? Self.automaticBurstDuration) + let delay = activeDuration * (1 - dutyCycle) / dutyCycle + return Decision(action: .runAfter(delay), targetDutyCycle: dutyCycle) + } +} diff --git a/Sources/CodexBar/PreferencesSpendDashboardPane.swift b/Sources/CodexBar/PreferencesSpendDashboardPane.swift index 83e7f31038..e55aa37bb4 100644 --- a/Sources/CodexBar/PreferencesSpendDashboardPane.swift +++ b/Sources/CodexBar/PreferencesSpendDashboardPane.swift @@ -27,6 +27,23 @@ func spendDashboardCoverageText(covered: Int, requested: Int) -> String { "\(L("Coverage")): \(codexBarLocalizedInteger(covered)) / \(codexBarLocalizedInteger(requested))" } +func codexCostCatchUpProgressText(_ activity: CodexCostCatchUpActivity) -> String { + if activity.totalBytes > 0 { + let processed = ByteCountFormatter.string( + fromByteCount: activity.processedBytes, + countStyle: .file) + let total = ByteCountFormatter.string( + fromByteCount: activity.totalBytes, + countStyle: .file) + return "\(processed) / \(total)" + } + if activity.totalFiles > 0 { + return "\(codexBarLocalizedInteger(activity.completedFiles)) / " + + codexBarLocalizedInteger(activity.totalFiles) + } + return L("Loading…") +} + enum SpendDashboardModelHistoryPresentation: Equatable { case unavailable case empty @@ -48,6 +65,7 @@ struct SpendDashboardPane: View { @Bindable var settings: SettingsStore @Bindable var store: UsageStore @State private var controller: SpendDashboardController + @State private var isVisible = false init(settings: SettingsStore, store: UsageStore) { self.settings = settings @@ -61,6 +79,7 @@ struct SpendDashboardPane: View { ScrollView { VStack(alignment: .leading, spacing: 18) { self.header + self.codexCostCatchUpPanel self.content self.provenance self.shareAction @@ -69,13 +88,26 @@ struct SpendDashboardPane: View { } .background(FocusResigningBackground()) .onAppear { + self.isVisible = true self.controller.refreshDateWindow() self.controller.update(configuration: self.configuration) + if !self.controller.isRefreshing { + self.synchronizeCodexCostCatchUp() + } } .onChange(of: self.configuration) { _, configuration in self.controller.update(configuration: configuration) + if self.isVisible, !self.controller.isRefreshing { + self.synchronizeCodexCostCatchUp() + } + } + .onChange(of: self.controller.isRefreshing) { _, isRefreshing in + if self.isVisible, !isRefreshing { + self.synchronizeCodexCostCatchUp() + } } .onDisappear { + self.isVisible = false self.controller.stop() } .onReceive(NotificationCenter.default.publisher(for: .NSCalendarDayChanged)) { _ in @@ -124,6 +156,131 @@ struct SpendDashboardPane: View { } } + @ViewBuilder + private var codexCostCatchUpPanel: some View { + if let activity = self.store.spendDashboardCodexCostCatchUpActivity, + activity.phase != .complete + { + SpendDashboardPanel { + VStack(alignment: .leading, spacing: 10) { + HStack(spacing: 8) { + Label( + self.codexCostCatchUpTitle(activity), + systemImage: activity.phase == .paused ? "pause.circle" : "externaldrive") + .font(.headline) + Spacer() + Text(codexCostCatchUpProgressText(activity)) + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + } + + if let progress = activity.fractionCompleted { + ProgressView(value: progress) + } else if activity.phase == .indexing { + ProgressView() + .controlSize(.small) + } + + if let staleSnapshotUpdatedAt = activity.staleSnapshotUpdatedAt { + HStack(spacing: 6) { + Label(L("stale data"), systemImage: "clock.badge.exclamationmark") + Text(L( + "Updated relative %@", + staleSnapshotUpdatedAt.relativeDescription())) + } + .font(.caption.weight(.medium)) + .foregroundStyle(.orange) + } + + Text(self.codexCostCatchUpDetail(activity)) + .font(.caption) + .foregroundStyle(.secondary) + + HStack { + if activity.pauseReason == .user + || activity.pauseReason == .noProgress + || self.codexCostCatchUpHasError(activity) + { + Button(L("Refresh")) { + self.startCodexCostCatchUp(mode: .automatic) + } + } else if activity.mode == .automatic { + Button(L("Finish now")) { + self.startCodexCostCatchUp(mode: .accelerated) + } + } else { + Button(L("Continue in background")) { + self.startCodexCostCatchUp(mode: .automatic) + } + } + + if activity.pauseReason != .user, + activity.pauseReason != .noProgress, + !self.codexCostCatchUpHasError(activity) + { + Button(L("Cancel")) { + self.store.stopSpendDashboardCodexCostCatchUp() + } + } + } + .controlSize(.small) + } + } + } + } + + private func codexCostCatchUpHasError(_ activity: CodexCostCatchUpActivity) -> Bool { + if case .error = activity.pauseReason { + return true + } + return false + } + + private func synchronizeCodexCostCatchUp() { + self.store.synchronizeSpendDashboardCodexCostCatchUp( + accounts: self.codexSpendScanRequests) + } + + private func startCodexCostCatchUp(mode: CodexCostCatchUpMode) { + self.store.startSpendDashboardCodexCostCatchUpIfNeeded( + accounts: self.codexSpendScanRequests, + mode: mode) + } + + private var codexSpendScanRequests: [CodexSpendScanRequest] { + guard self.configuration.costUsageEnabled, + self.configuration.providerIDs.contains(UsageProvider.codex.rawValue) + else { return [] } + return SpendDashboardSource.codexRequests(settings: self.settings, store: self.store) + } + + private func codexCostCatchUpTitle(_ activity: CodexCostCatchUpActivity) -> String { + let prefix = L("Local estimated history") + switch activity.phase { + case .indexing: + return "\(prefix) · \(L("Refreshing"))" + case .paused: + return "\(prefix) · \(L("Inactive"))" + case .complete: + return "\(prefix) · \(L("Done"))" + } + } + + private func codexCostCatchUpDetail(_ activity: CodexCostCatchUpActivity) -> String { + switch activity.pauseReason { + case .lowPower: + L("Battery Saver") + case .thermal, .user: + L("Inactive") + case .noProgress: + L("Error") + case let .error(message): + L("cost_status_error", L("Cost"), message) + case nil: + L("Estimated from local Codex logs for the selected account.") + } + } + @ViewBuilder private var content: some View { if !self.settings.costUsageEnabled { diff --git a/Sources/CodexBar/Resources/ar.lproj/Localizable.strings b/Sources/CodexBar/Resources/ar.lproj/Localizable.strings index 656e84539a..f69c45be7a 100644 --- a/Sources/CodexBar/Resources/ar.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/ar.lproj/Localizable.strings @@ -1353,3 +1353,5 @@ "Cost today unavailable" = "تكلفة اليوم: غير متوفر"; "30-day cost unavailable" = "تكلفة 30 يوماً: غير متوفر"; "Resets" = "إعادات الضبط"; +"Finish now" = "إنهاء الآن"; +"Continue in background" = "المتابعة في الخلفية"; diff --git a/Sources/CodexBar/Resources/ca.lproj/Localizable.strings b/Sources/CodexBar/Resources/ca.lproj/Localizable.strings index 1921f5a94d..f03fd8d72c 100644 --- a/Sources/CodexBar/Resources/ca.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/ca.lproj/Localizable.strings @@ -1352,3 +1352,5 @@ "Cost today unavailable" = "Cost d’avui: No disponible"; "30-day cost unavailable" = "Cost de 30 dies: No disponible"; "Resets" = "Reinicis"; +"Finish now" = "Finalitza ara"; +"Continue in background" = "Continua en segon pla"; diff --git a/Sources/CodexBar/Resources/de.lproj/Localizable.strings b/Sources/CodexBar/Resources/de.lproj/Localizable.strings index 2986c311d4..ecf0c320f7 100644 --- a/Sources/CodexBar/Resources/de.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/de.lproj/Localizable.strings @@ -1350,3 +1350,5 @@ "Cost today unavailable" = "Kosten heute: Nicht verfügbar"; "30-day cost unavailable" = "Kosten 30 Tage: Nicht verfügbar"; "Resets" = "Zurücksetzungen"; +"Finish now" = "Jetzt abschließen"; +"Continue in background" = "Im Hintergrund fortfahren"; diff --git a/Sources/CodexBar/Resources/en.lproj/Localizable.strings b/Sources/CodexBar/Resources/en.lproj/Localizable.strings index 18b42e8801..f3506f060f 100644 --- a/Sources/CodexBar/Resources/en.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/en.lproj/Localizable.strings @@ -1354,3 +1354,5 @@ "Cost today unavailable" = "Cost today unavailable"; "30-day cost unavailable" = "30-day cost unavailable"; "Resets" = "Resets"; +"Finish now" = "Finish now"; +"Continue in background" = "Continue in background"; diff --git a/Sources/CodexBar/Resources/es.lproj/Localizable.strings b/Sources/CodexBar/Resources/es.lproj/Localizable.strings index 656ac7975c..31cb13b6af 100644 --- a/Sources/CodexBar/Resources/es.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/es.lproj/Localizable.strings @@ -1348,3 +1348,5 @@ "Cost today unavailable" = "Coste de hoy: No disponible"; "30-day cost unavailable" = "Coste de 30 días: No disponible"; "Resets" = "Reinicios"; +"Finish now" = "Finalizar ahora"; +"Continue in background" = "Continuar en segundo plano"; diff --git a/Sources/CodexBar/Resources/fa.lproj/Localizable.strings b/Sources/CodexBar/Resources/fa.lproj/Localizable.strings index cdea6d030b..e0e65fcefe 100644 --- a/Sources/CodexBar/Resources/fa.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/fa.lproj/Localizable.strings @@ -1353,3 +1353,5 @@ "Cost today unavailable" = "هزینهٔ امروز: در دسترس نیست"; "30-day cost unavailable" = "هزینهٔ ۳۰ روز: در دسترس نیست"; "Resets" = "بازنشانی‌ها"; +"Finish now" = "اکنون تمام شود"; +"Continue in background" = "ادامه در پس‌زمینه"; diff --git a/Sources/CodexBar/Resources/fr.lproj/Localizable.strings b/Sources/CodexBar/Resources/fr.lproj/Localizable.strings index ff583450d9..23cee81947 100644 --- a/Sources/CodexBar/Resources/fr.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/fr.lproj/Localizable.strings @@ -1349,3 +1349,5 @@ "Cost today unavailable" = "Coût aujourd’hui: Indisponible"; "30-day cost unavailable" = "Coût sur 30 j: Indisponible"; "Resets" = "Réinitialisations"; +"Finish now" = "Terminer maintenant"; +"Continue in background" = "Continuer en arrière-plan"; diff --git a/Sources/CodexBar/Resources/gl.lproj/Localizable.strings b/Sources/CodexBar/Resources/gl.lproj/Localizable.strings index 43bd695894..35a39c656a 100644 --- a/Sources/CodexBar/Resources/gl.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/gl.lproj/Localizable.strings @@ -1349,3 +1349,5 @@ "Cost today unavailable" = "Custo de hoxe: Non dispoñible"; "30-day cost unavailable" = "Custo de 30 días: Non dispoñible"; "Resets" = "Reinicios"; +"Finish now" = "Rematar agora"; +"Continue in background" = "Continuar en segundo plano"; diff --git a/Sources/CodexBar/Resources/id.lproj/Localizable.strings b/Sources/CodexBar/Resources/id.lproj/Localizable.strings index 275d7b769e..cb4e2ecdd4 100644 --- a/Sources/CodexBar/Resources/id.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/id.lproj/Localizable.strings @@ -1353,3 +1353,5 @@ "Cost today unavailable" = "Biaya hari ini: Tidak tersedia"; "30-day cost unavailable" = "Biaya 30 hari: Tidak tersedia"; "Resets" = "Reset"; +"Finish now" = "Selesaikan sekarang"; +"Continue in background" = "Lanjutkan di latar belakang"; diff --git a/Sources/CodexBar/Resources/it.lproj/Localizable.strings b/Sources/CodexBar/Resources/it.lproj/Localizable.strings index 6ce92c2a94..e6aa45036d 100644 --- a/Sources/CodexBar/Resources/it.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/it.lproj/Localizable.strings @@ -1353,3 +1353,5 @@ "Cost today unavailable" = "Costo oggi: Non disponibile"; "30-day cost unavailable" = "Costo 30 gg: Non disponibile"; "Resets" = "Ripristini"; +"Finish now" = "Completa ora"; +"Continue in background" = "Continua in background"; diff --git a/Sources/CodexBar/Resources/ja.lproj/Localizable.strings b/Sources/CodexBar/Resources/ja.lproj/Localizable.strings index 7b6911423d..173424bf90 100644 --- a/Sources/CodexBar/Resources/ja.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/ja.lproj/Localizable.strings @@ -1350,3 +1350,5 @@ "Cost today unavailable" = "今日のコスト: 利用不可"; "30-day cost unavailable" = "30日間のコスト: 利用不可"; "Resets" = "リセット"; +"Finish now" = "今すぐ完了"; +"Continue in background" = "バックグラウンドで続行"; diff --git a/Sources/CodexBar/Resources/ko.lproj/Localizable.strings b/Sources/CodexBar/Resources/ko.lproj/Localizable.strings index 7458d95078..39841d832e 100644 --- a/Sources/CodexBar/Resources/ko.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/ko.lproj/Localizable.strings @@ -1317,3 +1317,5 @@ "Cost today unavailable" = "오늘 비용: 사용할 수 없음"; "30-day cost unavailable" = "30일 비용: 사용할 수 없음"; "Resets" = "재설정"; +"Finish now" = "지금 완료"; +"Continue in background" = "백그라운드에서 계속"; diff --git a/Sources/CodexBar/Resources/nl.lproj/Localizable.strings b/Sources/CodexBar/Resources/nl.lproj/Localizable.strings index cbe4af8954..fe88a4e1fa 100644 --- a/Sources/CodexBar/Resources/nl.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/nl.lproj/Localizable.strings @@ -1349,3 +1349,5 @@ "Cost today unavailable" = "Kosten vandaag: Niet beschikbaar"; "30-day cost unavailable" = "Kosten 30 dagen: Niet beschikbaar"; "Resets" = "Resets"; +"Finish now" = "Nu voltooien"; +"Continue in background" = "Doorgaan op de achtergrond"; diff --git a/Sources/CodexBar/Resources/pl.lproj/Localizable.strings b/Sources/CodexBar/Resources/pl.lproj/Localizable.strings index 9958e32f9e..c77d818520 100644 --- a/Sources/CodexBar/Resources/pl.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/pl.lproj/Localizable.strings @@ -1353,3 +1353,5 @@ "Cost today unavailable" = "Koszt dzisiaj: Niedostępne"; "30-day cost unavailable" = "Koszt 30 dni: Niedostępne"; "Resets" = "Resety"; +"Finish now" = "Zakończ teraz"; +"Continue in background" = "Kontynuuj w tle"; diff --git a/Sources/CodexBar/Resources/pt-BR.lproj/Localizable.strings b/Sources/CodexBar/Resources/pt-BR.lproj/Localizable.strings index 434ba9148c..618e0d541a 100644 --- a/Sources/CodexBar/Resources/pt-BR.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/pt-BR.lproj/Localizable.strings @@ -1350,3 +1350,5 @@ "Cost today unavailable" = "Custo hoje: Indisponível"; "30-day cost unavailable" = "Custo em 30 dias: Indisponível"; "Resets" = "Redefinições"; +"Finish now" = "Concluir agora"; +"Continue in background" = "Continuar em segundo plano"; diff --git a/Sources/CodexBar/Resources/ru.lproj/Localizable.strings b/Sources/CodexBar/Resources/ru.lproj/Localizable.strings index a1015d75f2..afbc65dc47 100644 --- a/Sources/CodexBar/Resources/ru.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/ru.lproj/Localizable.strings @@ -1351,3 +1351,5 @@ "Cost today unavailable" = "Расход сегодня: Недоступно"; "30-day cost unavailable" = "Расход за 30 дней: Недоступно"; "Resets" = "Сбросы"; +"Finish now" = "Завершить сейчас"; +"Continue in background" = "Продолжить в фоновом режиме"; diff --git a/Sources/CodexBar/Resources/sv.lproj/Localizable.strings b/Sources/CodexBar/Resources/sv.lproj/Localizable.strings index 541451b974..a0eedea938 100644 --- a/Sources/CodexBar/Resources/sv.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/sv.lproj/Localizable.strings @@ -1348,3 +1348,5 @@ "Cost today unavailable" = "Kostnad idag: Inte tillgänglig"; "30-day cost unavailable" = "Kostnad 30 dagar: Inte tillgänglig"; "Resets" = "Återställningar"; +"Finish now" = "Slutför nu"; +"Continue in background" = "Fortsätt i bakgrunden"; diff --git a/Sources/CodexBar/Resources/th.lproj/Localizable.strings b/Sources/CodexBar/Resources/th.lproj/Localizable.strings index 4e0f2ff782..f554fa058c 100644 --- a/Sources/CodexBar/Resources/th.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/th.lproj/Localizable.strings @@ -1353,3 +1353,5 @@ "Cost today unavailable" = "ค่าใช้จ่ายวันนี้: ไม่พร้อมใช้งาน"; "30-day cost unavailable" = "ค่าใช้จ่าย 30 วัน: ไม่พร้อมใช้งาน"; "Resets" = "การรีเซ็ต"; +"Finish now" = "เสร็จสิ้นตอนนี้"; +"Continue in background" = "ทำต่อในเบื้องหลัง"; diff --git a/Sources/CodexBar/Resources/tr.lproj/Localizable.strings b/Sources/CodexBar/Resources/tr.lproj/Localizable.strings index 48a5096e45..1da750b498 100644 --- a/Sources/CodexBar/Resources/tr.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/tr.lproj/Localizable.strings @@ -1351,3 +1351,5 @@ "Cost today unavailable" = "Bugünkü maliyet: Kullanılamıyor"; "30-day cost unavailable" = "30 günlük maliyet: Kullanılamıyor"; "Resets" = "Sıfırlamalar"; +"Finish now" = "Şimdi tamamla"; +"Continue in background" = "Arka planda devam et"; diff --git a/Sources/CodexBar/Resources/uk.lproj/Localizable.strings b/Sources/CodexBar/Resources/uk.lproj/Localizable.strings index 1915ea7bf9..0b33ba5e64 100644 --- a/Sources/CodexBar/Resources/uk.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/uk.lproj/Localizable.strings @@ -1349,3 +1349,5 @@ "Cost today unavailable" = "Вартість сьогодні: Недоступний"; "30-day cost unavailable" = "Вартість за 30 днів: Недоступний"; "Resets" = "Скидання"; +"Finish now" = "Завершити зараз"; +"Continue in background" = "Продовжити у фоновому режимі"; diff --git a/Sources/CodexBar/Resources/vi.lproj/Localizable.strings b/Sources/CodexBar/Resources/vi.lproj/Localizable.strings index 02c9b8fdf3..d285581972 100644 --- a/Sources/CodexBar/Resources/vi.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/vi.lproj/Localizable.strings @@ -1350,3 +1350,5 @@ "Cost today unavailable" = "Chi phí hôm nay: Không có sẵn"; "30-day cost unavailable" = "Chi phí 30 ngày: Không có sẵn"; "Resets" = "Lần đặt lại"; +"Finish now" = "Hoàn tất ngay"; +"Continue in background" = "Tiếp tục trong nền"; diff --git a/Sources/CodexBar/Resources/zh-Hans.lproj/Localizable.strings b/Sources/CodexBar/Resources/zh-Hans.lproj/Localizable.strings index 06fc0d4755..a96f096797 100644 --- a/Sources/CodexBar/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/zh-Hans.lproj/Localizable.strings @@ -1325,3 +1325,5 @@ "Cost today unavailable" = "今日费用: 不可用"; "30-day cost unavailable" = "30 天费用: 不可用"; "Resets" = "重置"; +"Finish now" = "立即完成"; +"Continue in background" = "转为后台运行"; diff --git a/Sources/CodexBar/Resources/zh-Hant.lproj/Localizable.strings b/Sources/CodexBar/Resources/zh-Hant.lproj/Localizable.strings index ec01ed18f3..0d02b24518 100644 --- a/Sources/CodexBar/Resources/zh-Hant.lproj/Localizable.strings +++ b/Sources/CodexBar/Resources/zh-Hant.lproj/Localizable.strings @@ -1380,3 +1380,5 @@ "Cost today unavailable" = "今日費用: 無法使用"; "30-day cost unavailable" = "30 天費用: 無法使用"; "Resets" = "重設"; +"Finish now" = "立即完成"; +"Continue in background" = "在背景繼續"; diff --git a/Sources/CodexBar/SpendDashboardController.swift b/Sources/CodexBar/SpendDashboardController.swift index 8b7de621e3..f56a80d75d 100644 --- a/Sources/CodexBar/SpendDashboardController.swift +++ b/Sources/CodexBar/SpendDashboardController.swift @@ -265,14 +265,12 @@ enum SpendDashboardSource { for account in request.codexRequests { let sourceID = "codex:\(account.id)" do { - guard self.currentAuthFingerprint(for: account) == account.authFingerprint else { + guard self.codexAuthFingerprintMatches(account) else { failedSourceIDs.insert(sourceID) invalidatedSourceIDs.insert(sourceID) continue } - let cacheRoot = UsageStore.costUsageCacheDirectory() - .appendingPathComponent("accounts", isDirectory: true) - .appendingPathComponent(account.cacheIdentity, isDirectory: true) + let cacheRoot = self.codexCacheRoot(for: account) let snapshot = try await codexSnapshotLoader(CodexSpendSnapshotLoadContext( account: account, cacheRoot: cacheRoot, @@ -282,7 +280,7 @@ enum SpendDashboardSource { refreshPricingInBackground: false, includePiSessions: false)) try Task.checkCancellation() - guard self.currentAuthFingerprint(for: account) == account.authFingerprint else { + guard self.codexAuthFingerprintMatches(account) else { failedSourceIDs.insert(sourceID) invalidatedSourceIDs.insert(sourceID) continue @@ -304,7 +302,7 @@ enum SpendDashboardSource { } } let lateInvalidatedSourceIDs = Set(request.codexRequests.compactMap { account in - self.currentAuthFingerprint(for: account) == account.authFingerprint + self.codexAuthFingerprintMatches(account) ? nil : "codex:\(account.id)" }) @@ -366,7 +364,11 @@ enum SpendDashboardSource { settings: SettingsStore, store: UsageStore) -> [String] { - ["settings:\(settings.configRevision)"] + providers.compactMap { provider in + var revisions = ["settings:\(settings.configRevision)"] + if providers.contains(.codex) { + revisions.append("codex-dashboard:\(store.spendDashboardCodexCostCatchUpRevision)") + } + revisions += providers.compactMap { provider in guard provider != .codex else { return nil } let current = store.tokenSnapshotPublicationForCurrentProviderConfig(for: provider) guard let current else { return "\(provider.rawValue):unavailable" } @@ -375,6 +377,7 @@ enum SpendDashboardSource { } return "\(provider.rawValue):snapshot:\(current.publicationRevision):\(self.snapshotRevision(snapshot))" } + return revisions } private static func snapshotRevision(_ snapshot: CostUsageTokenSnapshot) -> String { @@ -494,6 +497,16 @@ enum SpendDashboardSource { SHA256.hash(data: value).map { String(format: "%02x", $0) }.joined() } + static func codexCacheRoot(for request: CodexSpendScanRequest) -> URL { + UsageStore.costUsageCacheDirectory() + .appendingPathComponent("accounts", isDirectory: true) + .appendingPathComponent(request.cacheIdentity, isDirectory: true) + } + + static func codexAuthFingerprintMatches(_ request: CodexSpendScanRequest) -> Bool { + self.currentAuthFingerprint(for: request) == request.authFingerprint + } + private static func currentAuthFingerprint(for request: CodexSpendScanRequest) -> String? { let current = CodexAuthFingerprint.fingerprint(homePath: request.homePath) return request.authFileWasReadable ? current : current ?? request.authFingerprint diff --git a/Sources/CodexBar/UsageStore+CodexCostCatchUp.swift b/Sources/CodexBar/UsageStore+CodexCostCatchUp.swift new file mode 100644 index 0000000000..39de988e04 --- /dev/null +++ b/Sources/CodexBar/UsageStore+CodexCostCatchUp.swift @@ -0,0 +1,352 @@ +import CodexBarCore +import Foundation + +private struct CodexCostCatchUpContext { + let token: UUID + let codexHomePath: String? + let historyDays: Int + let scopeSignature: String + let providerConfigRevision: UInt64 + let costUsageSettingsRevision: UInt64 +} + +extension UsageStore { + func startCodexCostCatchUpIfNeeded(afterRefreshing provider: UsageProvider) { + guard provider == .codex else { return } + self.startCodexCostCatchUpIfNeeded(mode: .automatic) + } + + func startCodexCostCatchUpIfNeeded(mode: CodexCostCatchUpMode = .automatic) { + let scope = self.tokenCostScope(for: .codex) + let scopeSignature = self.tokenSnapshotScopeSignature(for: .codex) + if self.codexCostCatchUpTask != nil, + self.codexCostCatchUpScopeSignature == scopeSignature + { + guard self.codexCostCatchUpMode != mode else { return } + self.codexCostCatchUpMode = mode + // Never cancel a pass while it may be committing a resume checkpoint. The new mode + // applies immediately after that bounded pass completes. + if self.codexCostCatchUpPassIsRunning { + return + } + } + + self.cancelCodexCostCatchUp() + let token = UUID() + let context = CodexCostCatchUpContext( + token: token, + codexHomePath: scope.codexHomePath, + historyDays: self.settings.costUsageHistoryDays, + scopeSignature: scopeSignature, + providerConfigRevision: self.settings.providerConfigRevision(for: .codex), + costUsageSettingsRevision: self.settings.costUsageSettingsRevision) + self.codexCostCatchUpToken = token + self.codexCostCatchUpScopeSignature = scopeSignature + self.codexCostCatchUpMode = mode + self.codexCostCatchUpStopRequested = false + self.codexCostCatchUpPassIsRunning = false + let priority: TaskPriority = mode == .accelerated ? .utility : .background + self.codexCostCatchUpTask = Task(priority: priority) { @MainActor [weak self] in + guard let self else { return } + defer { + if self.codexCostCatchUpToken == token { + self.codexCostCatchUpTask = nil + self.codexCostCatchUpToken = nil + self.codexCostCatchUpScopeSignature = nil + } + } + await self.runCodexCostCatchUp(context: context) + } + } + + func cancelCodexCostCatchUp() { + self.codexCostCatchUpTask?.cancel() + self.codexCostCatchUpTask = nil + self.codexCostCatchUpToken = nil + self.codexCostCatchUpScopeSignature = nil + self.codexCostCatchUpStopRequested = false + self.codexCostCatchUpPassIsRunning = false + self.codexCostCatchUpActivity = nil + } + + func startAcceleratedCodexCostCatchUp() { + self.startCodexCostCatchUpIfNeeded(mode: .accelerated) + } + + func returnCodexCostCatchUpToBackground() { + self.startCodexCostCatchUpIfNeeded(mode: .automatic) + } + + func stopCodexCostCatchUp() { + guard self.codexCostCatchUpTask != nil else { return } + self.codexCostCatchUpStopRequested = true + guard !self.codexCostCatchUpPassIsRunning else { return } + if let activity = self.codexCostCatchUpActivity { + self.codexCostCatchUpActivity = CodexCostCatchUpActivity( + phase: .paused, + mode: activity.mode, + processedBytes: activity.processedBytes, + totalBytes: activity.totalBytes, + completedFiles: activity.completedFiles, + totalFiles: activity.totalFiles, + pauseReason: .user, + staleSnapshotUpdatedAt: activity.staleSnapshotUpdatedAt) + } + self.codexCostCatchUpTask?.cancel() + self.codexCostCatchUpTask = nil + self.codexCostCatchUpToken = nil + self.codexCostCatchUpScopeSignature = nil + } + + private func runCodexCostCatchUp(context: CodexCostCatchUpContext) async { + while self.codexCostCatchUpContextIsCurrent(context) { + var status = await self.loadCodexCostCatchUpStatus(codexHomePath: context.codexHomePath) + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: status.pending ? .indexing : .complete) + var didAdvance = false + var previousActiveDuration: TimeInterval? + while status.pending { + do { + guard self.codexCostCatchUpContextIsCurrent(context) else { return } + if self.codexCostCatchUpStopRequested { + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: .paused, + pauseReason: .user) + return + } + + let decision = self.codexCostCatchUpDecision( + mode: self.codexCostCatchUpMode, + previousActiveDuration: previousActiveDuration) + switch decision.action { + case let .pause(delay, reason): + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: .paused, + pauseReason: reason) + try await self.sleepBetweenCodexCostCatchUpPasses(seconds: delay) + continue + case let .runAfter(delay): + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: .indexing) + try await self.sleepBetweenCodexCostCatchUpPasses(seconds: delay) + } + + try Task.checkCancellation() + guard self.codexCostCatchUpContextIsCurrent(context) else { return } + if self.codexCostCatchUpStopRequested { + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: .paused, + pauseReason: .user) + return + } + + let passStartedAt = ContinuousClock.now + self.codexCostCatchUpPassIsRunning = true + let nextStatus: CostUsageFetcher.CodexScanCatchUpStatus + do { + nextStatus = try await self.advanceCodexCostCatchUp( + now: Date(), + codexHomePath: context.codexHomePath, + historyDays: context.historyDays) + self.codexCostCatchUpPassIsRunning = false + } catch { + self.codexCostCatchUpPassIsRunning = false + throw error + } + let passDuration = ContinuousClock.now - passStartedAt + let durationComponents = passDuration.components + previousActiveDuration = max( + 0, + Double(durationComponents.seconds) + + Double(durationComponents.attoseconds) / 1_000_000_000_000_000_000) + didAdvance = true + guard self.codexCostCatchUpContextIsCurrent(context) else { return } + self.publishCodexCostCatchUpActivity( + status: nextStatus, + context: context, + phase: nextStatus.pending ? .indexing : .complete) + if self.codexCostCatchUpStopRequested { + self.publishCodexCostCatchUpActivity( + status: nextStatus, + context: context, + phase: .paused, + pauseReason: .user) + return + } + if nextStatus.pending, nextStatus.progressKey == status.progressKey { + self.publishCodexCostCatchUpActivity( + status: nextStatus, + context: context, + phase: .paused, + pauseReason: .noProgress) + CodexBarLog.logger(LogCategories.tokenCost).warning( + "Codex cost catch-up stopped because a bounded pass made no progress") + return + } + status = nextStatus + } catch is CancellationError { + return + } catch { + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: .paused, + pauseReason: .error(error.localizedDescription)) + CodexBarLog.logger(LogCategories.tokenCost).warning( + "Codex cost catch-up stopped after error: \(error.localizedDescription)") + return + } + } + + guard self.codexCostCatchUpContextIsCurrent(context) else { return } + guard didAdvance else { + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: .complete) + return + } + do { + status = try await self.publishStableCodexCostCatchUpSnapshot(context: context) + guard status.pending else { return } + } catch is CancellationError { + return + } catch { + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: .paused, + pauseReason: .error(error.localizedDescription)) + CodexBarLog.logger(LogCategories.tokenCost).warning( + "Codex cost catch-up final snapshot failed: \(error.localizedDescription)") + return + } + } + } + + private func publishStableCodexCostCatchUpSnapshot( + context: CodexCostCatchUpContext) async throws -> CostUsageFetcher.CodexScanCatchUpStatus + { + let now = Date() + let snapshot = try await self.loadTokenUsageSnapshot( + provider: .codex, + force: true, + now: now, + codexHomePath: context.codexHomePath, + historyDays: context.historyDays) + try Task.checkCancellation() + guard self.codexCostCatchUpContextIsCurrent(context) else { + throw CancellationError() + } + + self.lastTokenFetchAt[.codex] = now + self.lastTokenFetchScope[.codex] = context.scopeSignature + if snapshot.daily.isEmpty, snapshot.meteredCostUSD == nil { + self.publishConfirmedEmptyTokenSnapshot(for: .codex) + self.tokenErrors[.codex] = Self.tokenCostNoDataMessage(for: .codex) + } else { + self.publishTokenSnapshot(snapshot, for: .codex) + self.tokenErrors[.codex] = nil + } + self.tokenFailureGates[.codex]?.recordSuccess() + self.persistWidgetSnapshot(reason: "token-usage-catch-up") + + let status = await self.loadCodexCostCatchUpStatus(codexHomePath: context.codexHomePath) + self.publishCodexCostCatchUpActivity( + status: status, + context: context, + phase: status.pending ? .indexing : .complete) + return status + } + + private func codexCostCatchUpContextIsCurrent(_ context: CodexCostCatchUpContext) -> Bool { + !Task.isCancelled + && self.codexCostCatchUpToken == context.token + && self.settings.providerConfigRevision(for: .codex) == context.providerConfigRevision + && self.settings.costUsageSettingsRevision == context.costUsageSettingsRevision + && self.settings.costUsageHistoryDays == context.historyDays + && self.settings.isCostUsageEffectivelyEnabled(for: .codex) + && self.isEnabled(.codex) + && self.tokenCostScope(for: .codex).codexHomePath == context.codexHomePath + && self.tokenSnapshotScopeSignature(for: .codex) == context.scopeSignature + } + + private func loadCodexCostCatchUpStatus( + codexHomePath: String?) async -> CostUsageFetcher.CodexScanCatchUpStatus + { + if let override = self._test_codexCostCatchUpStatusOverride { + return await override(codexHomePath) + } + return await self.costUsageFetcher.codexScanCatchUpStatus(codexHomePath: codexHomePath) + } + + private func advanceCodexCostCatchUp( + now: Date, + codexHomePath: String?, + historyDays: Int) async throws -> CostUsageFetcher.CodexScanCatchUpStatus + { + if let override = self._test_codexCostCatchUpAdvanceOverride { + return try await override(now, codexHomePath, historyDays) + } + return try await self.costUsageFetcher.advanceCodexScanCatchUp( + now: now, + codexHomePath: codexHomePath, + historyDays: historyDays) + } + + private func codexCostCatchUpDecision( + mode: CodexCostCatchUpMode, + previousActiveDuration: TimeInterval?) -> CodexCostCatchUpPolicy.Decision + { + let resourceState = self._test_codexCostCatchUpResourceStateOverride?() ?? ( + powerSource: CodexCostCatchUpPowerSource.current(), + lowPowerModeEnabled: ProcessInfo.processInfo.isLowPowerModeEnabled, + thermalState: ProcessInfo.processInfo.thermalState) + return CodexCostCatchUpPolicy().decision(for: .init( + mode: mode, + previousActiveDuration: previousActiveDuration, + powerSource: resourceState.powerSource, + lowPowerModeEnabled: resourceState.lowPowerModeEnabled, + thermalState: resourceState.thermalState)) + } + + private func publishCodexCostCatchUpActivity( + status: CostUsageFetcher.CodexScanCatchUpStatus, + context: CodexCostCatchUpContext, + phase: CodexCostCatchUpActivity.Phase, + pauseReason: CodexCostCatchUpPauseReason? = nil) + { + guard self.codexCostCatchUpToken == context.token else { return } + self.codexCostCatchUpActivity = CodexCostCatchUpActivity( + phase: phase, + mode: self.codexCostCatchUpMode, + processedBytes: status.processedBytes, + totalBytes: status.totalBytes, + completedFiles: status.completedFiles, + totalFiles: status.totalFiles, + pauseReason: pauseReason, + staleSnapshotUpdatedAt: status.staleSnapshotUpdatedAt) + } + + private func sleepBetweenCodexCostCatchUpPasses(seconds: TimeInterval) async throws { + if let override = self._test_codexCostCatchUpSleepOverride { + try await override(max(0, seconds)) + return + } + guard seconds > 0 else { + await Task.yield() + return + } + try await Task.sleep(for: .seconds(seconds)) + } +} diff --git a/Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift b/Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift new file mode 100644 index 0000000000..0ee8c911ec --- /dev/null +++ b/Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift @@ -0,0 +1,361 @@ +import CodexBarCore +import Foundation + +private struct SpendDashboardCodexCostCatchUpContext { + let token: UUID + let accounts: [CodexSpendScanRequest] + let historyDays: Int + let scopeSignature: String + let providerConfigRevision: UInt64 + let costUsageSettingsRevision: UInt64 +} + +extension UsageStore { + func synchronizeSpendDashboardCodexCostCatchUp(accounts: [CodexSpendScanRequest]) { + let mode = self.spendDashboardCodexCostCatchUpTask == nil + ? .automatic + : self.spendDashboardCodexCostCatchUpMode + self.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts, mode: mode) + } + + func startSpendDashboardCodexCostCatchUpIfNeeded( + accounts: [CodexSpendScanRequest], + mode: CodexCostCatchUpMode = .automatic) + { + let accounts = Self.uniqueSpendDashboardCodexAccounts(accounts) + guard !accounts.isEmpty, + self.settings.isCostUsageEffectivelyEnabled(for: .codex), + self.isEnabled(.codex) + else { + self.cancelSpendDashboardCodexCostCatchUp() + return + } + + let scopeSignature = accounts + .map { "\($0.id)|\($0.cacheIdentity)" } + .joined(separator: "\u{0}") + if self.spendDashboardCodexCostCatchUpTask != nil, + self.spendDashboardCodexCostCatchUpScopeSignature == scopeSignature + { + guard self.spendDashboardCodexCostCatchUpMode != mode else { return } + self.spendDashboardCodexCostCatchUpMode = mode + // A bounded parser pass may be committing a resume checkpoint. Let it finish and + // apply the new mode before scheduling the next account instead of cancelling it. + if self.spendDashboardCodexCostCatchUpPassIsRunning { + return + } + } + + self.cancelSpendDashboardCodexCostCatchUp() + let token = UUID() + let context = SpendDashboardCodexCostCatchUpContext( + token: token, + accounts: accounts, + historyDays: SpendDashboardSource.scanDays, + scopeSignature: scopeSignature, + providerConfigRevision: self.settings.providerConfigRevision(for: .codex), + costUsageSettingsRevision: self.settings.costUsageSettingsRevision) + self.spendDashboardCodexCostCatchUpToken = token + self.spendDashboardCodexCostCatchUpScopeSignature = scopeSignature + self.spendDashboardCodexCostCatchUpMode = mode + self.spendDashboardCodexCostCatchUpStopRequested = false + self.spendDashboardCodexCostCatchUpPassIsRunning = false + let priority: TaskPriority = mode == .accelerated ? .utility : .background + self.spendDashboardCodexCostCatchUpTask = Task(priority: priority) { @MainActor [weak self] in + guard let self else { return } + defer { + if self.spendDashboardCodexCostCatchUpToken == token { + self.spendDashboardCodexCostCatchUpTask = nil + self.spendDashboardCodexCostCatchUpToken = nil + self.spendDashboardCodexCostCatchUpScopeSignature = nil + } + } + await self.runSpendDashboardCodexCostCatchUp(context: context) + } + } + + func stopSpendDashboardCodexCostCatchUp() { + guard self.spendDashboardCodexCostCatchUpTask != nil else { return } + self.spendDashboardCodexCostCatchUpStopRequested = true + guard !self.spendDashboardCodexCostCatchUpPassIsRunning else { return } + if let activity = self.spendDashboardCodexCostCatchUpActivity { + self.spendDashboardCodexCostCatchUpActivity = CodexCostCatchUpActivity( + phase: .paused, + mode: activity.mode, + processedBytes: activity.processedBytes, + totalBytes: activity.totalBytes, + completedFiles: activity.completedFiles, + totalFiles: activity.totalFiles, + pauseReason: .user, + staleSnapshotUpdatedAt: activity.staleSnapshotUpdatedAt) + } + self.spendDashboardCodexCostCatchUpTask?.cancel() + self.spendDashboardCodexCostCatchUpTask = nil + self.spendDashboardCodexCostCatchUpToken = nil + self.spendDashboardCodexCostCatchUpScopeSignature = nil + } + + func cancelSpendDashboardCodexCostCatchUp() { + self.spendDashboardCodexCostCatchUpTask?.cancel() + self.spendDashboardCodexCostCatchUpTask = nil + self.spendDashboardCodexCostCatchUpToken = nil + self.spendDashboardCodexCostCatchUpScopeSignature = nil + self.spendDashboardCodexCostCatchUpStopRequested = false + self.spendDashboardCodexCostCatchUpPassIsRunning = false + self.spendDashboardCodexCostCatchUpActivity = nil + } + + private func runSpendDashboardCodexCostCatchUp( + context: SpendDashboardCodexCostCatchUpContext) async + { + var statuses = await self.loadSpendDashboardCodexCostCatchUpStatuses(context.accounts) + guard self.spendDashboardCodexCostCatchUpContextIsCurrent(context) else { return } + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: Self.spendDashboardCodexCatchUpIsPending(statuses) ? .indexing : .complete) + + var didChangeCache = false + var previousActiveDuration: TimeInterval? + var stalledCacheIdentities: Set = [] + while Self.spendDashboardCodexCatchUpIsPending(statuses) { + do { + guard self.spendDashboardCodexCostCatchUpContextIsCurrent(context) else { return } + if self.spendDashboardCodexCostCatchUpStopRequested { + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: .paused, + pauseReason: .user) + self.publishSpendDashboardCodexCostCatchUpRevisionIfNeeded(didChangeCache) + return + } + + guard let account = context.accounts.first(where: { + statuses[$0.cacheIdentity]?.pending == true + && !stalledCacheIdentities.contains($0.cacheIdentity) + }) else { + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: .paused, + pauseReason: .noProgress) + self.publishSpendDashboardCodexCostCatchUpRevisionIfNeeded(didChangeCache) + CodexBarLog.logger(LogCategories.tokenCost).warning( + "Spend Dashboard Codex cost catch-up stopped because all pending account caches stalled") + return + } + + let decision = self.spendDashboardCodexCostCatchUpDecision( + mode: self.spendDashboardCodexCostCatchUpMode, + previousActiveDuration: previousActiveDuration) + switch decision.action { + case let .pause(delay, reason): + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: .paused, + pauseReason: reason) + try await self.sleepBetweenSpendDashboardCodexCostCatchUpPasses(seconds: delay) + continue + case let .runAfter(delay): + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: .indexing) + try await self.sleepBetweenSpendDashboardCodexCostCatchUpPasses(seconds: delay) + } + + try Task.checkCancellation() + guard self.spendDashboardCodexCostCatchUpContextIsCurrent(context) else { return } + if self.spendDashboardCodexCostCatchUpStopRequested { + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: .paused, + pauseReason: .user) + self.publishSpendDashboardCodexCostCatchUpRevisionIfNeeded(didChangeCache) + return + } + + let previousStatus = statuses[account.cacheIdentity] + let passStartedAt = ContinuousClock.now + self.spendDashboardCodexCostCatchUpPassIsRunning = true + let nextStatus: CostUsageFetcher.CodexScanCatchUpStatus + do { + nextStatus = try await self.advanceSpendDashboardCodexCostCatchUp( + account: account, + now: Date(), + historyDays: context.historyDays) + self.spendDashboardCodexCostCatchUpPassIsRunning = false + } catch { + self.spendDashboardCodexCostCatchUpPassIsRunning = false + throw error + } + previousActiveDuration = Self.spendDashboardCodexCatchUpDuration( + since: passStartedAt) + didChangeCache = didChangeCache || nextStatus.progressKey != previousStatus?.progressKey + statuses[account.cacheIdentity] = nextStatus + if nextStatus.pending, + nextStatus.progressKey == previousStatus?.progressKey + { + stalledCacheIdentities.insert(account.cacheIdentity) + } else { + stalledCacheIdentities.remove(account.cacheIdentity) + } + + guard self.spendDashboardCodexCostCatchUpContextIsCurrent(context) else { return } + let isPending = Self.spendDashboardCodexCatchUpIsPending(statuses) + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: isPending ? .indexing : .complete) + if self.spendDashboardCodexCostCatchUpStopRequested { + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: .paused, + pauseReason: .user) + self.publishSpendDashboardCodexCostCatchUpRevisionIfNeeded(didChangeCache) + return + } + } catch is CancellationError { + return + } catch { + self.publishSpendDashboardCodexCostCatchUpActivity( + statuses: statuses, + context: context, + phase: .paused, + pauseReason: .error(error.localizedDescription)) + self.publishSpendDashboardCodexCostCatchUpRevisionIfNeeded(didChangeCache) + CodexBarLog.logger(LogCategories.tokenCost).warning( + "Spend Dashboard Codex cost catch-up stopped after error: \(error.localizedDescription)") + return + } + } + + self.publishSpendDashboardCodexCostCatchUpRevisionIfNeeded(didChangeCache) + } + + private func spendDashboardCodexCostCatchUpContextIsCurrent( + _ context: SpendDashboardCodexCostCatchUpContext) -> Bool + { + !Task.isCancelled + && self.spendDashboardCodexCostCatchUpToken == context.token + && self.spendDashboardCodexCostCatchUpScopeSignature == context.scopeSignature + && self.settings.providerConfigRevision(for: .codex) == context.providerConfigRevision + && self.settings.costUsageSettingsRevision == context.costUsageSettingsRevision + && self.settings.isCostUsageEffectivelyEnabled(for: .codex) + && self.isEnabled(.codex) + && context.accounts.allSatisfy(SpendDashboardSource.codexAuthFingerprintMatches) + } + + private func loadSpendDashboardCodexCostCatchUpStatuses( + _ accounts: [CodexSpendScanRequest]) async -> [String: CostUsageFetcher.CodexScanCatchUpStatus] + { + var statuses: [String: CostUsageFetcher.CodexScanCatchUpStatus] = [:] + for account in accounts { + if let override = self._test_spendDashboardCodexCostCatchUpStatusOverride { + statuses[account.cacheIdentity] = await override(account) + } else { + statuses[account.cacheIdentity] = await CostUsageFetcher( + cacheRoot: SpendDashboardSource.codexCacheRoot(for: account)) + .codexScanCatchUpStatus(codexHomePath: account.homePath) + } + } + return statuses + } + + private func advanceSpendDashboardCodexCostCatchUp( + account: CodexSpendScanRequest, + now: Date, + historyDays: Int) async throws -> CostUsageFetcher.CodexScanCatchUpStatus + { + if let override = self._test_spendDashboardCodexCostCatchUpAdvanceOverride { + return try await override(account, now, historyDays) + } + return try await CostUsageFetcher(cacheRoot: SpendDashboardSource.codexCacheRoot(for: account)) + .advanceCodexScanCatchUp( + now: now, + codexHomePath: account.homePath, + historyDays: historyDays) + } + + private func spendDashboardCodexCostCatchUpDecision( + mode: CodexCostCatchUpMode, + previousActiveDuration: TimeInterval?) -> CodexCostCatchUpPolicy.Decision + { + let resourceState = self._test_spendDashboardCodexCostCatchUpResourceStateOverride?() ?? ( + powerSource: CodexCostCatchUpPowerSource.current(), + lowPowerModeEnabled: ProcessInfo.processInfo.isLowPowerModeEnabled, + thermalState: ProcessInfo.processInfo.thermalState) + return CodexCostCatchUpPolicy().decision(for: .init( + mode: mode, + previousActiveDuration: previousActiveDuration, + powerSource: resourceState.powerSource, + lowPowerModeEnabled: resourceState.lowPowerModeEnabled, + thermalState: resourceState.thermalState)) + } + + private func publishSpendDashboardCodexCostCatchUpActivity( + statuses: [String: CostUsageFetcher.CodexScanCatchUpStatus], + context: SpendDashboardCodexCostCatchUpContext, + phase: CodexCostCatchUpActivity.Phase, + pauseReason: CodexCostCatchUpPauseReason? = nil) + { + guard self.spendDashboardCodexCostCatchUpToken == context.token else { return } + let values = context.accounts.compactMap { statuses[$0.cacheIdentity] } + let hasIndeterminatePendingStatus = values.contains { + $0.pending && $0.totalBytes == 0 && $0.totalFiles == 0 + } + self.spendDashboardCodexCostCatchUpActivity = CodexCostCatchUpActivity( + phase: phase, + mode: self.spendDashboardCodexCostCatchUpMode, + processedBytes: hasIndeterminatePendingStatus ? 0 : values.reduce(0) { $0 + $1.processedBytes }, + totalBytes: hasIndeterminatePendingStatus ? 0 : values.reduce(0) { $0 + $1.totalBytes }, + completedFiles: hasIndeterminatePendingStatus ? 0 : values.reduce(0) { $0 + $1.completedFiles }, + totalFiles: hasIndeterminatePendingStatus ? 0 : values.reduce(0) { $0 + $1.totalFiles }, + pauseReason: pauseReason, + staleSnapshotUpdatedAt: values.compactMap(\.staleSnapshotUpdatedAt).min()) + } + + private func publishSpendDashboardCodexCostCatchUpRevisionIfNeeded(_ didChangeCache: Bool) { + guard didChangeCache else { return } + self.spendDashboardCodexCostCatchUpRevision &+= 1 + } + + private func sleepBetweenSpendDashboardCodexCostCatchUpPasses(seconds: TimeInterval) async throws { + if let override = self._test_spendDashboardCodexCostCatchUpSleepOverride { + try await override(max(0, seconds)) + return + } + guard seconds > 0 else { + await Task.yield() + return + } + try await Task.sleep(for: .seconds(seconds)) + } + + private static func uniqueSpendDashboardCodexAccounts( + _ accounts: [CodexSpendScanRequest]) -> [CodexSpendScanRequest] + { + var seen: Set = [] + return accounts.filter { seen.insert($0.cacheIdentity).inserted } + } + + private static func spendDashboardCodexCatchUpIsPending( + _ statuses: [String: CostUsageFetcher.CodexScanCatchUpStatus]) -> Bool + { + statuses.values.contains(where: \.pending) + } + + private static func spendDashboardCodexCatchUpDuration( + since start: ContinuousClock.Instant) -> TimeInterval + { + let components = (ContinuousClock.now - start).components + return max( + 0, + Double(components.seconds) + + Double(components.attoseconds) / 1_000_000_000_000_000_000) + } +} diff --git a/Sources/CodexBar/UsageStore+TokenCost.swift b/Sources/CodexBar/UsageStore+TokenCost.swift index a55a14e72c..b6a202c1d8 100644 --- a/Sources/CodexBar/UsageStore+TokenCost.swift +++ b/Sources/CodexBar/UsageStore+TokenCost.swift @@ -213,8 +213,10 @@ extension UsageStore { return Task { @MainActor [weak self] in guard let self else { return } guard self.tokenSnapshotPublicationForCurrentProviderConfig(for: .codex) == nil else { return } - let result: (snapshot: CostUsageTokenSnapshot, lastRefreshAt: Date?)? = if let override = self - ._test_cachedCodexTokenSnapshotLoaderOverride + let result: ( + snapshot: CostUsageTokenSnapshot, + lastRefreshAt: Date?, + staleSnapshotUpdatedAt: Date?)? = if let override = self._test_cachedCodexTokenSnapshotLoaderOverride { await override(now, scope.codexHomePath, historyDays) } else { @@ -222,7 +224,12 @@ extension UsageStore { now: now, codexHomePath: scope.codexHomePath, historyDays: historyDays) - .map { (snapshot: $0.snapshot, lastRefreshAt: $0.lastRefreshAt) } + .map { + ( + snapshot: $0.snapshot, + lastRefreshAt: $0.lastRefreshAt, + staleSnapshotUpdatedAt: $0.staleSnapshotUpdatedAt) + } } guard let result else { @@ -243,6 +250,9 @@ extension UsageStore { } self.installCachedTokenSnapshot(result.snapshot, for: .codex) self.tokenErrors[.codex] = nil + if result.staleSnapshotUpdatedAt != nil { + self.startCodexCostCatchUpIfNeeded() + } if let tokenFetchTTL = self.tokenFetchTTL, let lastRefreshAt = result.lastRefreshAt, now.timeIntervalSince(lastRefreshAt) >= 0, diff --git a/Sources/CodexBar/UsageStore.swift b/Sources/CodexBar/UsageStore.swift index 19c9488717..644df63d85 100644 --- a/Sources/CodexBar/UsageStore.swift +++ b/Sources/CodexBar/UsageStore.swift @@ -23,6 +23,7 @@ extension UsageStore { _ = self.tokenSnapshots _ = self.tokenErrors _ = self.tokenRefreshInFlight + _ = self.codexCostCatchUpActivity _ = self.credits _ = self.lastCreditsError _ = self.openAIDashboard @@ -183,6 +184,9 @@ final class UsageStore { var tokenSnapshotPublicationRevisions: [UsageProvider: UInt64] = [:] var tokenErrors: [UsageProvider: String] = [:] var tokenRefreshInFlight: Set = [] + var codexCostCatchUpActivity: CodexCostCatchUpActivity? + var spendDashboardCodexCostCatchUpActivity: CodexCostCatchUpActivity? + var spendDashboardCodexCostCatchUpRevision: UInt64 = 0 var credits: CreditsSnapshot? var lastCreditsError: String? var openAIDashboard: OpenAIDashboardSnapshot? @@ -260,7 +264,34 @@ final class UsageStore { @ObservationIgnored var _test_cachedCodexTokenSnapshotLoaderOverride: (@MainActor ( Date, String?, - Int) async -> (snapshot: CostUsageTokenSnapshot, lastRefreshAt: Date?)?)? + Int) async -> ( + snapshot: CostUsageTokenSnapshot, + lastRefreshAt: Date?, + staleSnapshotUpdatedAt: Date?)?)? + @ObservationIgnored var _test_codexCostCatchUpStatusOverride: (@MainActor ( + String?) async -> CostUsageFetcher.CodexScanCatchUpStatus)? + @ObservationIgnored var _test_codexCostCatchUpAdvanceOverride: (@MainActor ( + Date, + String?, + Int) async throws -> CostUsageFetcher.CodexScanCatchUpStatus)? + @ObservationIgnored var _test_codexCostCatchUpSleepOverride: (@MainActor ( + TimeInterval) async throws -> Void)? + @ObservationIgnored var _test_codexCostCatchUpResourceStateOverride: (@MainActor () -> ( + powerSource: CodexCostCatchUpPowerSource, + lowPowerModeEnabled: Bool, + thermalState: ProcessInfo.ThermalState))? + @ObservationIgnored var _test_spendDashboardCodexCostCatchUpStatusOverride: (@MainActor ( + CodexSpendScanRequest) async -> CostUsageFetcher.CodexScanCatchUpStatus)? + @ObservationIgnored var _test_spendDashboardCodexCostCatchUpAdvanceOverride: (@MainActor ( + CodexSpendScanRequest, + Date, + Int) async throws -> CostUsageFetcher.CodexScanCatchUpStatus)? + @ObservationIgnored var _test_spendDashboardCodexCostCatchUpSleepOverride: (@MainActor ( + TimeInterval) async throws -> Void)? + @ObservationIgnored var _test_spendDashboardCodexCostCatchUpResourceStateOverride: (@MainActor () -> ( + powerSource: CodexCostCatchUpPowerSource, + lowPowerModeEnabled: Bool, + thermalState: ProcessInfo.ThermalState))? @ObservationIgnored var _test_providerStatusFetchOverride: (@MainActor ( UsageProvider) async throws -> ProviderStatus)? @ObservationIgnored var _test_forcedRefreshEnrichmentWaitObserver: (@MainActor () -> Void)? @@ -308,6 +339,18 @@ final class UsageStore { @ObservationIgnored var tokenRefreshSequenceToken: UUID? @ObservationIgnored var tokenRefreshSequenceProvider: UsageProvider? @ObservationIgnored var tokenRefreshRetryProviders: Set = [] + @ObservationIgnored var codexCostCatchUpTask: Task? + @ObservationIgnored var codexCostCatchUpToken: UUID? + @ObservationIgnored var codexCostCatchUpScopeSignature: String? + @ObservationIgnored var codexCostCatchUpMode: CodexCostCatchUpMode = .automatic + @ObservationIgnored var codexCostCatchUpStopRequested = false + @ObservationIgnored var codexCostCatchUpPassIsRunning = false + @ObservationIgnored var spendDashboardCodexCostCatchUpTask: Task? + @ObservationIgnored var spendDashboardCodexCostCatchUpToken: UUID? + @ObservationIgnored var spendDashboardCodexCostCatchUpScopeSignature: String? + @ObservationIgnored var spendDashboardCodexCostCatchUpMode: CodexCostCatchUpMode = .automatic + @ObservationIgnored var spendDashboardCodexCostCatchUpStopRequested = false + @ObservationIgnored var spendDashboardCodexCostCatchUpPassIsRunning = false @ObservationIgnored var forcedRefreshEnrichmentTask: Task? @ObservationIgnored var forcedRefreshEnrichmentToken: UUID? @ObservationIgnored var pendingForcedRefreshEnrichmentTask: Task? @@ -846,6 +889,7 @@ final class UsageStore { self.timerTask?.cancel() self.tokenTimerTask?.cancel() self.tokenRefreshSequenceTask?.cancel() + self.codexCostCatchUpTask?.cancel() self.forcedRefreshEnrichmentTask?.cancel() self.pendingForcedRefreshEnrichmentTask?.cancel() self.requiredRefreshTask?.cancel() @@ -1462,6 +1506,7 @@ extension UsageStore { return } self.lastTokenFetchScope[provider] = completedCostScopeSignature + self.startCodexCostCatchUpIfNeeded(afterRefreshing: provider) guard !snapshot.daily.isEmpty || snapshot.meteredCostUSD != nil else { self.publishConfirmedEmptyTokenSnapshot(for: provider) @@ -1524,6 +1569,10 @@ extension UsageStore { } private func resetTokenUsageState(for provider: UsageProvider) { + if provider == .codex { + self.cancelCodexCostCatchUp() + self.cancelSpendDashboardCodexCostCatchUp() + } self.clearTokenSnapshot(for: provider) self.tokenErrors[provider] = nil self.tokenFailureGates[provider]?.reset() diff --git a/Sources/CodexBarCore/CostUsageFetcher.swift b/Sources/CodexBarCore/CostUsageFetcher.swift index 226d061364..e15afb6d77 100644 --- a/Sources/CodexBarCore/CostUsageFetcher.swift +++ b/Sources/CodexBarCore/CostUsageFetcher.swift @@ -28,9 +28,40 @@ public enum CostUsageError: LocalizedError, Sendable { // swiftlint:disable:next type_body_length public struct CostUsageFetcher: Sendable { + private static let codexAutomaticScanDurationPerRefresh: TimeInterval = 2 + package struct CachedCodexTokenSnapshotResult: Sendable { package let snapshot: CostUsageTokenSnapshot package let lastRefreshAt: Date? + package let staleSnapshotUpdatedAt: Date? + } + + package struct CodexScanCatchUpStatus: Sendable, Equatable { + package let pending: Bool + package let progressKey: String + package let processedBytes: Int64 + package let totalBytes: Int64 + package let completedFiles: Int + package let totalFiles: Int + package let staleSnapshotUpdatedAt: Date? + + package init( + pending: Bool, + progressKey: String, + processedBytes: Int64 = 0, + totalBytes: Int64 = 0, + completedFiles: Int = 0, + totalFiles: Int = 0, + staleSnapshotUpdatedAt: Date? = nil) + { + self.pending = pending + self.progressKey = progressKey + self.processedBytes = max(0, processedBytes) + self.totalBytes = max(0, totalBytes) + self.completedFiles = max(0, completedFiles) + self.totalFiles = max(0, totalFiles) + self.staleSnapshotUpdatedAt = staleSnapshotUpdatedAt + } } private let scannerOptions: CostUsageScanner.Options? @@ -194,6 +225,97 @@ public struct CostUsageFetcher: Sendable { self.scannerOptions } + package func codexScanCatchUpStatus( + codexHomePath: String? = nil) async -> CodexScanCatchUpStatus + { + let options = Self.resolvedScannerOptions( + self.scannerOptionsOverride(), + provider: .codex, + codexHomePath: codexHomePath) + return await (try? CostUsageScanExecutor.run { checkCancellation in + try checkCancellation() + return Self.codexScanCatchUpStatus(options: options) + }) ?? CodexScanCatchUpStatus(pending: false, progressKey: "unavailable") + } + + package func advanceCodexScanCatchUp( + now: Date = Date(), + codexHomePath: String? = nil, + historyDays: Int = 30) async throws -> CodexScanCatchUpStatus + { + var options = Self.resolvedScannerOptions( + self.scannerOptionsOverride(), + provider: .codex, + codexHomePath: codexHomePath) + options.forceRescan = false + options.refreshMinIntervalSeconds = 0 + options.maxCodexScanDurationPerRefresh = Self.codexAutomaticScanDurationPerRefresh + let clampedHistoryDays = max(1, min(365, historyDays)) + let since = options.calendar.date( + byAdding: .day, + value: -(clampedHistoryDays - 1), + to: now) ?? now + let scanOptions = options + return try await CostUsageScanExecutor.run { checkCancellation in + _ = try CostUsageScanner.loadDailyReportCancellable( + provider: .codex, + since: since, + until: now, + now: now, + options: scanOptions, + checkCancellation: checkCancellation) + try checkCancellation() + return Self.codexScanCatchUpStatus(options: scanOptions) + } + } + + private static func codexScanCatchUpStatus( + options: CostUsageScanner.Options) -> CodexScanCatchUpStatus + { + let roots = CostUsageScanner.codexSessionsRoots(options: options) + let rootsFingerprint = CostUsageScanner.codexRootsFingerprint(options: options) + let loadedCache = CostUsageCacheIO.loadCodexForMigration( + cacheRoot: options.cacheRoot, + calendar: options.calendar) + let cache = loadedCache.cache + guard cache.roots == rootsFingerprint else { + if let incompatibleCache = loadedCache.incompatibleCache, + incompatibleCache.roots == rootsFingerprint + { + let staleSnapshotUpdatedAt: Date? = if incompatibleCache.lastScanUnixMs > 0 { + Date(timeIntervalSince1970: TimeInterval(incompatibleCache.lastScanUnixMs) / 1000) + } else { + nil + } + return CodexScanCatchUpStatus( + pending: true, + progressKey: "producer-upgrade", + staleSnapshotUpdatedAt: staleSnapshotUpdatedAt) + } + return CodexScanCatchUpStatus(pending: false, progressKey: "scope-mismatch") + } + + let scoped = CostUsageScanner.codexCache(cache, scopedTo: roots) + var progressHasher = Hasher() + for (path, usage) in scoped.files.sorted(by: { $0.key < $1.key }) { + progressHasher.combine(path) + progressHasher.combine(usage.codexScanFileId) + progressHasher.combine(usage.parsedBytes) + progressHasher.combine(usage.size) + progressHasher.combine(usage.codexScanComplete) + } + let hasIncompleteFile = scoped.files.values.contains { $0.codexScanComplete == false } + let pending = cache.codexScanCatchUpPending == true || hasIncompleteFile + return CodexScanCatchUpStatus( + pending: pending, + progressKey: "\(scoped.files.count):\(progressHasher.finalize())", + processedBytes: cache.codexScanProcessedBytes ?? 0, + totalBytes: cache.codexScanTotalBytes ?? 0, + completedFiles: cache.codexScanCompletedFiles ?? 0, + totalFiles: cache.codexScanTotalFiles ?? 0, + staleSnapshotUpdatedAt: pending ? cache.codexPreviousReport?.updatedAt : nil) + } + private static func resolvedScannerOptions( _ override: CostUsageScanner.Options?, provider: UsageProvider, @@ -263,14 +385,12 @@ public struct CostUsageFetcher: Sendable { cacheRoot: options.cacheRoot, client: modelsDevClient) - if provider == .vertexai { - options.claudeLogProviderFilter = allowVertexClaudeFallback ? .all : .vertexAIOnly - } else if provider == .claude { - options.claudeLogProviderFilter = .excludeVertexAI - } - if forceRefresh || bypassScannerDebounce { - options.refreshMinIntervalSeconds = 0 - } + Self.configureScannerRefresh( + &options, + provider: provider, + allowVertexClaudeFallback: allowVertexClaudeFallback, + forceRefresh: forceRefresh, + bypassScannerDebounce: bypassScannerDebounce) var resolvedPiOptions = overridePiScannerOptions ?? PiSessionCostScanner.Options() if resolvedPiOptions.cacheRoot == nil { resolvedPiOptions.cacheRoot = options.cacheRoot @@ -281,28 +401,97 @@ public struct CostUsageFetcher: Sendable { } let piOptions = resolvedPiOptions - try Task.checkCancellation() - // The corpus scans below are synchronous and can run for minutes on large session - // archives. They execute on the dedicated scan queue so they never occupy a cooperative - // pool thread; CostUsageScanExecutor bridges this task's cancellation into the - // scanner-level checks. let scanOptions = options - let scanResult = try await CostUsageScanExecutor.run { checkCancellation in + let localScanOptions = LocalTokenScanOptions( + allowVertexClaudeFallback: allowVertexClaudeFallback, + includePiSessions: includePiSessions, + shouldMergePiUsage: shouldMergePiUsage, + scanOptions: scanOptions, + piOptions: piOptions) + let scanResult = try await Self.loadLocalTokenScanResult( + provider: provider, + since: since, + now: now, + options: localScanOptions) + + if allowPricingRefresh, + retryUnknownPricing, + let request = Self.unknownPricingRefreshRequest( + provider: provider, + daily: scanResult.daily, + now: now, + cacheRoot: options.cacheRoot, + client: modelsDevClient), + await Self.refreshUnknownPricingIfNeeded(request, inBackground: refreshPricingInBackground) + { + return try await self.loadTokenSnapshot( + provider: provider, + environment: environment, + now: now, + forceRefresh: forceRefresh, + allowVertexClaudeFallback: allowVertexClaudeFallback, + codexHomePath: codexHomePath, + historyDays: historyDays, + cursorCookieHeaderOverride: cursorCookieHeaderOverride, + allowPricingRefresh: allowPricingRefresh, + refreshPricingInBackground: false, + includePiSessions: includePiSessions, + scannerOptions: options, + piScannerOptions: piOptions, + modelsDevClient: modelsDevClient, + retryUnknownPricing: false) + } + + return Self.tokenSnapshot( + from: scanResult.daily, + now: now, + historyDays: clampedHistoryDays, + calendar: scanOptions.calendar, + projects: scanResult.projects, + sessions: scanResult.sessions, + updatedAt: scanResult.staleSnapshotUpdatedAt) + } + + private struct LocalTokenScanResult: Sendable { + let daily: CostUsageDailyReport + let projects: [CostUsageProjectBreakdown] + let sessions: [CostUsageSessionBreakdown] + let staleSnapshotUpdatedAt: Date? + } + + private struct LocalTokenScanOptions: Sendable { + let allowVertexClaudeFallback: Bool + let includePiSessions: Bool + let shouldMergePiUsage: Bool + let scanOptions: CostUsageScanner.Options + let piOptions: PiSessionCostScanner.Options + } + + private static func loadLocalTokenScanResult( + provider: UsageProvider, + since: Date, + now: Date, + options: LocalTokenScanOptions) async throws -> LocalTokenScanResult + { + try Task.checkCancellation() + // These synchronous scans can run for minutes on large archives. The dedicated queue keeps + // them off the cooperative pool and bridges task cancellation into scanner-level checks. + return try await CostUsageScanExecutor.run { checkCancellation in var daily = try CostUsageScanner.loadDailyReportCancellable( provider: provider, since: since, until: now, now: now, - options: scanOptions, + options: options.scanOptions, checkCancellation: checkCancellation) try checkCancellation() if provider == .vertexai, - !allowVertexClaudeFallback, - scanOptions.claudeLogProviderFilter == .vertexAIOnly, + !options.allowVertexClaudeFallback, + options.scanOptions.claudeLogProviderFilter == .vertexAIOnly, daily.data.isEmpty { - var fallback = scanOptions + var fallback = options.scanOptions fallback.claudeLogProviderFilter = .all daily = try CostUsageScanner.loadDailyReportCancellable( provider: provider, @@ -317,30 +506,41 @@ public struct CostUsageFetcher: Sendable { var projects: [CostUsageProjectBreakdown] = [] var sessions: [CostUsageSessionBreakdown] = [] var piDaily: CostUsageDailyReport? + var staleSnapshotUpdatedAt: Date? if provider == .codex { - let roots = CostUsageScanner.codexSessionsRoots(options: scanOptions) + let roots = CostUsageScanner.codexSessionsRoots(options: options.scanOptions) let cache = CostUsageScanner.codexCache( - CostUsageCacheIO.load(provider: .codex, cacheRoot: scanOptions.cacheRoot), + CostUsageCacheIO.load(provider: .codex, cacheRoot: options.scanOptions.cacheRoot), scopedTo: roots) let range = CostUsageScanner.CostUsageDayRange( - since: since, until: now, calendar: scanOptions.calendar) - projects = CostUsageScanner.buildCodexProjectBreakdownsFromCache( + since: since, until: now, calendar: options.scanOptions.calendar) + if let previous = CostUsageScanner.codexPreviousReport( cache: cache, range: range, - modelsDevCacheRoot: scanOptions.cacheRoot) - sessions = CostUsageScanner.buildCodexSessionBreakdownsFromCache( - cache: cache, - range: range, - modelsDevCacheRoot: scanOptions.cacheRoot, - sessionRoots: roots) + rootsFingerprint: CostUsageScanner.codexRootsFingerprint(options: options.scanOptions)) + { + staleSnapshotUpdatedAt = previous.updatedAt + } else { + projects = CostUsageScanner.buildCodexProjectBreakdownsFromCache( + cache: cache, + range: range, + modelsDevCacheRoot: options.scanOptions.cacheRoot) + sessions = CostUsageScanner.buildCodexSessionBreakdownsFromCache( + cache: cache, + range: range, + modelsDevCacheRoot: options.scanOptions.cacheRoot, + sessionRoots: roots) + } } - if includePiSessions, provider == .claude || (provider == .codex && shouldMergePiUsage) { + if options.includePiSessions, + provider == .claude || (provider == .codex && options.shouldMergePiUsage) + { let piReport = try PiSessionCostScanner.loadDailyReportCancellable( provider: provider, since: since, until: now, now: now, - options: piOptions, + options: options.piOptions, checkCancellation: checkCancellation) try checkCancellation() if provider == .codex { @@ -355,44 +555,12 @@ public struct CostUsageFetcher: Sendable { sessions = [] } } - return (daily: daily, projects: projects, sessions: sessions) - } - - if allowPricingRefresh, - retryUnknownPricing, - let request = Self.unknownPricingRefreshRequest( - provider: provider, - daily: scanResult.daily, - now: now, - cacheRoot: options.cacheRoot, - client: modelsDevClient), - await Self.refreshUnknownPricingIfNeeded(request, inBackground: refreshPricingInBackground) - { - return try await self.loadTokenSnapshot( - provider: provider, - environment: environment, - now: now, - forceRefresh: forceRefresh, - allowVertexClaudeFallback: allowVertexClaudeFallback, - codexHomePath: codexHomePath, - historyDays: historyDays, - cursorCookieHeaderOverride: cursorCookieHeaderOverride, - allowPricingRefresh: allowPricingRefresh, - refreshPricingInBackground: false, - includePiSessions: includePiSessions, - scannerOptions: options, - piScannerOptions: piOptions, - modelsDevClient: modelsDevClient, - retryUnknownPricing: false) + return LocalTokenScanResult( + daily: daily, + projects: projects, + sessions: sessions, + staleSnapshotUpdatedAt: staleSnapshotUpdatedAt) } - - return Self.tokenSnapshot( - from: scanResult.daily, - now: now, - historyDays: clampedHistoryDays, - calendar: scanOptions.calendar, - projects: scanResult.projects, - sessions: scanResult.sessions) } private struct PricingRefreshOptions: Sendable { @@ -523,8 +691,12 @@ public struct CostUsageFetcher: Sendable { until: until, calendar: options.calendar) let roots = CostUsageScanner.codexSessionsRoots(options: options) + let rootsFingerprint = CostUsageScanner.codexRootsFingerprint(options: options) + let loadedCache = CostUsageCacheIO.loadCodexForMigration( + cacheRoot: options.cacheRoot, + calendar: options.calendar) let cache = CostUsageScanner.codexCache( - CostUsageCacheIO.load(provider: .codex, cacheRoot: options.cacheRoot), + loadedCache.cache, scopedTo: roots) var reports: [CostUsageDailyReport] = [] var projects: [CostUsageProjectBreakdown] = [] @@ -534,11 +706,22 @@ public struct CostUsageFetcher: Sendable { var nativeScanAt: Date? var scanTimes: [Date] = [] var piMerged = false + var staleSnapshotUpdatedAt: Date? - if cache.timeZoneIdentifier == range.calendar.timeZone.identifier, - !cache.days.isEmpty, - cache.roots == CostUsageScanner.codexRootsFingerprint(options: options), - !CostUsageScanner.requestedWindowExpandsCache(range: range, cache: cache) + if let previous = CostUsageScanner.codexPreviousReport( + cache: cache, + range: range, + rootsFingerprint: rootsFingerprint) + { + reports.append(previous.report) + staleSnapshotUpdatedAt = previous.updatedAt + if let updatedAt = previous.updatedAt { + scanTimes.append(updatedAt) + } + } else if cache.timeZoneIdentifier == range.calendar.timeZone.identifier, + !cache.days.isEmpty, + cache.roots == rootsFingerprint, + !CostUsageScanner.requestedWindowExpandsCache(range: range, cache: cache) { let daily = CostUsageScanner.buildCodexReportFromCache( cache: cache, @@ -563,6 +746,25 @@ public struct CostUsageFetcher: Sendable { modelsDevCacheRoot: options.cacheRoot)) } } + } else if let incompatibleCache = loadedCache.incompatibleCache, + incompatibleCache.timeZoneIdentifier == range.calendar.timeZone.identifier, + !incompatibleCache.days.isEmpty, + incompatibleCache.roots == rootsFingerprint, + !CostUsageScanner.requestedWindowExpandsCache(range: range, cache: incompatibleCache) + { + let daily = CostUsageScanner.buildCodexReportFromCache( + cache: incompatibleCache, + range: range, + modelsDevCacheRoot: options.cacheRoot) + if !daily.data.isEmpty { + reports.append(daily) + if incompatibleCache.lastScanUnixMs > 0 { + let scanAt = Date( + timeIntervalSince1970: TimeInterval(incompatibleCache.lastScanUnixMs) / 1000) + staleSnapshotUpdatedAt = scanAt + scanTimes.append(scanAt) + } + } } if let piResult = PiSessionCostScanner.loadCachedDailyReportResult( @@ -600,7 +802,8 @@ public struct CostUsageFetcher: Sendable { projects: Self.mergedProjectBreakdowns(projects), sessions: sessions, updatedAt: scanTimes.min()), - lastRefreshAt: piMerged ? nil : nativeScanAt) + lastRefreshAt: piMerged || staleSnapshotUpdatedAt != nil ? nil : nativeScanAt, + staleSnapshotUpdatedAt: staleSnapshotUpdatedAt) } return cachedSnapshot.flatMap(\.self) } @@ -798,6 +1001,43 @@ public struct CostUsageFetcher: Sendable { updatedAt: updatedAt ?? now) } + package static func resolvedCodexScanDurationPerRefresh( + provider: UsageProvider, + bypassScannerDebounce: Bool, + configuredDuration: TimeInterval?) -> TimeInterval? + { + guard provider == .codex, + bypassScannerDebounce, + configuredDuration == nil + else { return configuredDuration } + + // UsageStore refreshes set bypassScannerDebounce. Bound that first app scan too, + // so it can publish a partial snapshot and hand remaining work to the persistent + // catch-up loop instead of consuming the whole 512 MiB byte budget continuously. + return self.codexAutomaticScanDurationPerRefresh + } + + private static func configureScannerRefresh( + _ options: inout CostUsageScanner.Options, + provider: UsageProvider, + allowVertexClaudeFallback: Bool, + forceRefresh: Bool, + bypassScannerDebounce: Bool) + { + if provider == .vertexai { + options.claudeLogProviderFilter = allowVertexClaudeFallback ? .all : .vertexAIOnly + } else if provider == .claude { + options.claudeLogProviderFilter = .excludeVertexAI + } + if forceRefresh || bypassScannerDebounce { + options.refreshMinIntervalSeconds = 0 + } + options.maxCodexScanDurationPerRefresh = self.resolvedCodexScanDurationPerRefresh( + provider: provider, + bypassScannerDebounce: bypassScannerDebounce, + configuredDuration: options.maxCodexScanDurationPerRefresh) + } + private static func unknownProjectBreakdown(from daily: CostUsageDailyReport) -> CostUsageProjectBreakdown? { guard !daily.data.isEmpty else { return nil } return CostUsageProjectBreakdown( diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 6bcb4da8ac..f63c9e2f9b 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "f90ee3aa5b4f84a2" + static let value = "aa0b0865c496e548" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CodexSubagentRolloutShape.swift b/Sources/CodexBarCore/Vendored/CostUsage/CodexSubagentRolloutShape.swift index 56f5608a56..02811f9773 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CodexSubagentRolloutShape.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CodexSubagentRolloutShape.swift @@ -7,9 +7,10 @@ extension CostUsageScanner { } /// Subagent source is lineage evidence, not counter semantics. The first session metadata - /// owns leaf identity. Embedded ancestor metadata proves a copied prefix by itself; compact - /// rollouts need both the first-turn boundary and an exact parent snapshot match in the scanner. - /// Do not restore a blanket "all subagents are independent/inherited" rule. + /// owns leaf identity. Embedded ancestor metadata proves a copied prefix by itself. Compact + /// rollouts need a first-turn boundary plus either local `total - last` proof or an exact parent + /// snapshot match in the scanner. Do not restore a blanket "all subagents are + /// independent/inherited" rule. struct CodexSubagentRolloutShape { let counterSemantics: CodexSubagentCounterSemantics let ownedSuffix: CodexSubagentOwnedSuffix? @@ -24,6 +25,7 @@ extension CostUsageScanner { struct CodexSubagentOwnedSuffixCandidate { let ownedSuffix: CodexSubagentOwnedSuffix let parentTotalsAtBoundary: CostUsageCodexTotals + let isLocallyConfirmed: Bool } struct Observation { @@ -83,6 +85,7 @@ extension CostUsageScanner { var pendingTurnContext: (lineIndex: Int, baseline: CostUsageCodexTotals)? var ownedSuffix: CodexSubagentOwnedSuffix? var parentTotalsAtBoundary: CostUsageCodexTotals? + var locallyConfirmedBoundary = false var inspectedOwnedSuffixFirstTotal = false var observedAuthoritativeMetadata = false var observedTurnContext = false @@ -103,6 +106,7 @@ extension CostUsageScanner { // A later ancestor meta proves that any earlier candidate boundary was replay. ownedSuffix = nil parentTotalsAtBoundary = nil + locallyConfirmedBoundary = false inspectedOwnedSuffixFirstTotal = false } pendingTurnContext = nil @@ -128,6 +132,7 @@ extension CostUsageScanner { startLineIndex: pendingTurnContext.lineIndex, rawTotalsBaseline: pendingTurnContext.baseline) parentTotalsAtBoundary = pendingTurnContext.baseline + locallyConfirmedBoundary = false inspectedOwnedSuffixFirstTotal = false } pendingTurnContext = nil @@ -147,6 +152,15 @@ extension CostUsageScanner { ownedSuffix = Self.CodexSubagentOwnedSuffix( startLineIndex: suffix.startLineIndex, rawTotalsBaseline: .init(input: 0, cached: 0, output: 0)) + } else if let last, + let inferredBaseline = Self.subtract(last, from: total), + Self.totalsEqual(inferredBaseline, suffix.rawTotalsBaseline) + { + // Local delivery metadata is not part of copied model history. When + // the first owned cumulative row also proves total - last == the + // pre-boundary snapshot, the child can establish its inherited + // baseline without rereading the parent rollout. + locallyConfirmedBoundary = true } } if let total { @@ -167,7 +181,8 @@ extension CostUsageScanner { let candidate: CodexSubagentOwnedSuffixCandidate? = if let ownedSuffix, let parentTotalsAtBoundary { Self.CodexSubagentOwnedSuffixCandidate( ownedSuffix: ownedSuffix, - parentTotalsAtBoundary: parentTotalsAtBoundary) + parentTotalsAtBoundary: parentTotalsAtBoundary, + isLocallyConfirmed: locallyConfirmedBoundary) } else { nil } @@ -197,6 +212,26 @@ extension CostUsageScanner { totals.input > 0 || totals.cached > 0 || totals.output > 0 } + private static func subtract( + _ delta: CostUsageCodexTotals, + from total: CostUsageCodexTotals) -> CostUsageCodexTotals? + { + guard self.totalsAtLeast(total, delta) else { return nil } + let reasoning: Int? = if let totalReasoning = total.reasoning, + let deltaReasoning = delta.reasoning, + totalReasoning >= deltaReasoning + { + totalReasoning - deltaReasoning + } else { + nil + } + return CostUsageCodexTotals( + input: total.input - delta.input, + cached: total.cached - delta.cached, + output: total.output - delta.output, + reasoning: reasoning) + } + private static func normalizedSessionID(_ value: String?) -> String? { guard let value else { return nil } let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index e13e0bcd3b..2bdffa03d7 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -56,15 +56,42 @@ enum CostUsageCacheIO { return CostUsageCache() } + static func loadCodexForMigration( + cacheRoot: URL? = nil, + producerKey: String? = nil, + calendar: Calendar? = nil) -> CostUsageCodexCacheLoadResult + { + let url = self.cacheFileURL(provider: .codex, cacheRoot: cacheRoot) + guard let decoded = self.decodeCache(at: url) else { + return CostUsageCodexCacheLoadResult(cache: CostUsageCache(), incompatibleCache: nil) + } + if let calendar, decoded.timeZoneIdentifier != calendar.timeZone.identifier { + return CostUsageCodexCacheLoadResult(cache: CostUsageCache(), incompatibleCache: nil) + } + + let expectedProducerKey = producerKey ?? self.currentProducerKey(provider: .codex) + let compatibleProducerKeys = producerKey == nil ? self.compatibleCodexProducerKeys : [] + if decoded.producerKey == expectedProducerKey + || decoded.producerKey.map(compatibleProducerKeys.contains) == true + { + return CostUsageCodexCacheLoadResult(cache: decoded, incompatibleCache: nil) + } + + // Never reuse parser-dependent offsets or totals from an incompatible producer. The + // caller may still convert its last visible report into a compact, explicitly stale + // presentation while the current producer rebuilds from byte zero. + guard decoded.producerKey != nil else { + return CostUsageCodexCacheLoadResult(cache: CostUsageCache(), incompatibleCache: nil) + } + return CostUsageCodexCacheLoadResult(cache: CostUsageCache(), incompatibleCache: decoded) + } + private static func loadCache( at url: URL, expectedProducerKey: String?, compatibleProducerKeys: Set) -> CostUsageCache? { - guard let data = try? Data(contentsOf: url) else { return nil } - guard let decoded = try? JSONDecoder().decode(CostUsageCache.self, from: data) - else { return nil } - guard decoded.version == 1 else { return nil } + guard let decoded = self.decodeCache(at: url) else { return nil } if let expectedProducerKey { guard decoded.producerKey == expectedProducerKey || decoded.producerKey.map(compatibleProducerKeys.contains) == true @@ -73,6 +100,14 @@ enum CostUsageCacheIO { return decoded } + private static func decodeCache(at url: URL) -> CostUsageCache? { + guard let data = try? Data(contentsOf: url) else { return nil } + guard let decoded = try? JSONDecoder().decode(CostUsageCache.self, from: data) + else { return nil } + guard decoded.version == 1 else { return nil } + return decoded + } + static func save( provider: UsageProvider, cache: CostUsageCache, @@ -111,6 +146,11 @@ enum CostUsageCacheIO { } } +struct CostUsageCodexCacheLoadResult { + var cache: CostUsageCache + var incompatibleCache: CostUsageCache? +} + struct CostUsageCache: Codable { var version: Int = 1 var producerKey: String? @@ -123,6 +163,16 @@ struct CostUsageCache: Codable { var codexProjectMetadataVersion: Int? var codexPriorityTurnKeys: [String: String]? var codexPriorityTurnIDsByDay: [String: [String]]? + /// True when the last bounded scan left readable Codex work for a background catch-up pass. + var codexScanCatchUpPending: Bool? + var codexScanProcessedBytes: Int64? + var codexScanTotalBytes: Int64? + var codexScanCompletedFiles: Int? + var codexScanTotalFiles: Int? + /// Last user-visible report retained only while an incompatible or forced rebuild catches up. + var codexPreviousReport: CostUsageCodexPreviousReport? + /// Persistent session-id discovery and generation-scoped negative lookups for fork parents. + var codexSessionDiscovery: CostUsageCodexSessionDiscovery? /// filePath -> file usage var files: [String: CostUsageFileUsage] = [:] @@ -134,6 +184,191 @@ struct CostUsageCache: Codable { var roots: [String: Int64]? } +struct CostUsageCodexSessionDiscovery: Codable { + struct DirectoryStamp: Codable, Equatable { + var mtimeUnixMs: Int64 + var jsonlFileCount: Int + } + + struct FileStamp: Codable, Equatable { + var mtimeUnixMs: Int64 + var size: Int64 + var fileId: String? + } + + struct HeadScan: Codable { + var path: String + var offset: Int64 + var resumeState: CostUsageJsonl.ResumeState? + } + + var roots: [String] + var generation: String? + var directoryStamps: [String: DirectoryStamp] + var directoryPaths: [String] + var nextDirectoryIndex: Int + var filePaths: [String] + var nextFileIndex: Int + var fileStamps: [String: FileStamp] + var headScan: HeadScan? + var filePathBySessionId: [String: String] + var missingSessionIds: [String] + var pendingSessionIds: [String] + var validationDirectoryIndex: Int + var isComplete: Bool +} + +struct CostUsageCodexPreviousReport: Codable, Equatable { + struct ModelBreakdown: Codable, Equatable { + var modelName: String + var costUSD: Double? + var totalTokens: Int? + var requestCount: Int? + var standardCostUSD: Double? + var priorityCostUSD: Double? + var standardTokens: Int? + var priorityTokens: Int? + + init(_ breakdown: CostUsageDailyReport.ModelBreakdown) { + self.modelName = breakdown.modelName + self.costUSD = breakdown.costUSD + self.totalTokens = breakdown.totalTokens + self.requestCount = breakdown.requestCount + self.standardCostUSD = breakdown.standardCostUSD + self.priorityCostUSD = breakdown.priorityCostUSD + self.standardTokens = breakdown.standardTokens + self.priorityTokens = breakdown.priorityTokens + } + + var dailyReportValue: CostUsageDailyReport.ModelBreakdown { + CostUsageDailyReport.ModelBreakdown( + modelName: self.modelName, + costUSD: self.costUSD, + totalTokens: self.totalTokens, + requestCount: self.requestCount, + standardCostUSD: self.standardCostUSD, + priorityCostUSD: self.priorityCostUSD, + standardTokens: self.standardTokens, + priorityTokens: self.priorityTokens) + } + } + + struct Entry: Codable, Equatable { + var date: String + var inputTokens: Int? + var cacheReadTokens: Int? + var cacheCreationTokens: Int? + var outputTokens: Int? + var totalTokens: Int? + var requestCount: Int? + var costUSD: Double? + var modelsUsed: [String]? + var modelBreakdowns: [ModelBreakdown]? + + init(_ entry: CostUsageDailyReport.Entry) { + self.date = entry.date + self.inputTokens = entry.inputTokens + self.cacheReadTokens = entry.cacheReadTokens + self.cacheCreationTokens = entry.cacheCreationTokens + self.outputTokens = entry.outputTokens + self.totalTokens = entry.totalTokens + self.requestCount = entry.requestCount + self.costUSD = entry.costUSD + self.modelsUsed = entry.modelsUsed + self.modelBreakdowns = entry.modelBreakdowns?.map(ModelBreakdown.init) + } + + var dailyReportValue: CostUsageDailyReport.Entry { + CostUsageDailyReport.Entry( + date: self.date, + inputTokens: self.inputTokens, + outputTokens: self.outputTokens, + cacheReadTokens: self.cacheReadTokens, + cacheCreationTokens: self.cacheCreationTokens, + totalTokens: self.totalTokens, + requestCount: self.requestCount, + costUSD: self.costUSD, + modelsUsed: self.modelsUsed, + modelBreakdowns: self.modelBreakdowns?.map(\.dailyReportValue)) + } + } + + struct Summary: Codable, Equatable { + var totalInputTokens: Int? + var totalOutputTokens: Int? + var cacheReadTokens: Int? + var cacheCreationTokens: Int? + var totalTokens: Int? + var totalCostUSD: Double? + + init(_ summary: CostUsageDailyReport.Summary) { + self.totalInputTokens = summary.totalInputTokens + self.totalOutputTokens = summary.totalOutputTokens + self.cacheReadTokens = summary.cacheReadTokens + self.cacheCreationTokens = summary.cacheCreationTokens + self.totalTokens = summary.totalTokens + self.totalCostUSD = summary.totalCostUSD + } + + var dailyReportValue: CostUsageDailyReport.Summary { + CostUsageDailyReport.Summary( + totalInputTokens: self.totalInputTokens, + totalOutputTokens: self.totalOutputTokens, + cacheReadTokens: self.cacheReadTokens, + cacheCreationTokens: self.cacheCreationTokens, + totalTokens: self.totalTokens, + totalCostUSD: self.totalCostUSD) + } + } + + var data: [Entry] + var summary: Summary? + var updatedAtUnixMs: Int64 + var scanSinceKey: String? + var scanUntilKey: String? + var timeZoneIdentifier: String? + var roots: [String: Int64]? + + init?( + report: CostUsageDailyReport, + cache: CostUsageCache) + { + guard !report.data.isEmpty else { return nil } + self.data = report.data.map(Entry.init) + self.summary = report.summary.map(Summary.init) + self.updatedAtUnixMs = cache.lastScanUnixMs + self.scanSinceKey = cache.scanSinceKey + self.scanUntilKey = cache.scanUntilKey + self.timeZoneIdentifier = cache.timeZoneIdentifier + self.roots = cache.roots + } + + var report: CostUsageDailyReport { + CostUsageDailyReport( + data: self.data.map(\.dailyReportValue), + summary: self.summary?.dailyReportValue) + } + + var updatedAt: Date? { + guard self.updatedAtUnixMs > 0 else { return nil } + return Date(timeIntervalSince1970: TimeInterval(self.updatedAtUnixMs) / 1000) + } + + func matches( + scanSinceKey: String, + scanUntilKey: String, + timeZoneIdentifier: String, + roots: [String: Int64]) -> Bool + { + guard self.timeZoneIdentifier == timeZoneIdentifier, + self.roots == roots, + let cachedSince = self.scanSinceKey, + let cachedUntil = self.scanUntilKey + else { return false } + return scanSinceKey >= cachedSince && scanUntilKey <= cachedUntil + } +} + struct CostUsageFileUsage: Codable { var mtimeUnixMs: Int64 var size: Int64 @@ -165,14 +400,29 @@ struct CostUsageFileUsage: Codable { /// Refreshed by Codex normalization paths, never by sidecar cache validation. var codexWorkspaceContentFingerprint: String? var codexRows: [CostUsageScanner.CodexUsageRow]? + /// Compact token events used to resolve fork baselines without rereading an entire parent rollout. + var codexTokenSnapshots: [CostUsageCodexTokenSnapshot]? + /// Sparse accumulator states for bounded lookup inside `codexTokenSnapshots`. + var codexTokenCheckpoints: [CostUsageCodexTokenCheckpoint]? + /// Allows binary-search and early-stop lookup only when event timestamps follow file order. + var codexTokenTimestampsMonotonic: Bool? + /// Validates that the indexed JSONL prefix was not rewritten before an append. + var codexTokenIndexAnchor: CostUsageCodexTokenIndexAnchor? var claudeRows: [CostUsageScanner.ClaudeUsageRow]? - /// Identity and target size for an in-progress bounded Codex parse. + /// Identity and latest observed size for an in-progress bounded Codex parse. var codexScanFileId: String? var codexScanTargetSize: Int64? var codexScanComplete: Bool? var codexJSONLResumeState: CostUsageJsonl.ResumeState? /// Compact relevant events retained while a subagent rollout awaits full-shape classification. var codexBufferedSubagentLines: [CostUsageScanner.CodexBufferedFastLine]? + /// Parsed events retained when an ordinary fork is waiting for its parent baseline. + var codexBufferedUnresolvedForkLines: [CostUsageScanner.CodexBufferedFastLine]? + + var hasBufferedCodexForkRetryLines: Bool { + self.codexBufferedSubagentLines?.isEmpty == false + || self.codexBufferedUnresolvedForkLines?.isEmpty == false + } } struct CostUsageCodexSessionMetadata: Codable, Equatable { @@ -234,3 +484,45 @@ struct CostUsageCodexTotals: Codable, Equatable { self.reasoning = reasoning } } + +struct CostUsageCodexTokenSnapshot: Codable, Equatable { + var timestamp: String + var last: CostUsageCodexTotals? + var total: CostUsageCodexTotals? + var endOffset: Int64? + + init( + timestamp: String, + last: CostUsageCodexTotals?, + total: CostUsageCodexTotals?, + endOffset: Int64? = nil) + { + self.timestamp = timestamp + self.last = last + self.total = total + self.endOffset = endOffset + } +} + +struct CostUsageCodexTokenAccumulatorState: Codable, Equatable { + var countedTotals: CostUsageCodexTotals? + var rawTotalsBaseline: CostUsageCodexTotals? + var sawDivergentTotals: Bool + var rawTotalsWatermark: CostUsageCodexTotals? + var seenRawTotals: [CostUsageCodexTotals] + var sawInterleavedTotals: Bool +} + +struct CostUsageCodexTokenCheckpoint: Codable, Equatable { + /// Index of the last token event already folded into `state`. + var eventIndex: Int + var timestamp: String + var endOffset: Int64 + var state: CostUsageCodexTokenAccumulatorState +} + +struct CostUsageCodexTokenIndexAnchor: Codable, Equatable { + var indexedBytes: Int64 + var windowStart: Int64 + var sha256: String +} diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 4911826e5b..78d398b7c9 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -4,6 +4,20 @@ enum CostUsageJsonl { struct Line { let bytes: Data let wasTruncated: Bool + let startOffset: Int64 + let endOffset: Int64 + + init( + bytes: Data, + wasTruncated: Bool, + startOffset: Int64 = 0, + endOffset: Int64 = 0) + { + self.bytes = bytes + self.wasTruncated = wasTruncated + self.startOffset = startOffset + self.endOffset = endOffset + } } struct ResumeState: Codable { @@ -272,6 +286,7 @@ enum CostUsageJsonl { prefixBytes: prefixBytes, maxBytesToRead: maxBytesToRead, resumeState: nil, + shouldStop: nil, checkCancellation: checkCancellation, onLine: onLine).committedOffset } @@ -284,6 +299,7 @@ enum CostUsageJsonl { prefixBytes: Int, maxBytesToRead: Int64?, resumeState: ResumeState?, + shouldStop: ((Int64) -> Bool)? = nil, checkCancellation: (() throws -> Void)? = nil, onLine: (Line) -> Void) throws -> ScanProgress { @@ -320,9 +336,13 @@ enum CostUsageJsonl { } } - func flushLine() { + func flushLine(endOffset: Int64) { guard lineBytes > 0 else { return } - let line = Line(bytes: current, wasTruncated: truncated) + let line = Line( + bytes: current, + wasTruncated: truncated, + startOffset: lineStartOffset, + endOffset: endOffset) onLine(line) current.removeAll(keepingCapacity: true) lineBytes = 0 @@ -354,10 +374,13 @@ enum CostUsageJsonl { while true { try checkCancellation?() + if bytesRead > 0, shouldStop?(bytesRead) == true { + break + } let remaining = maxBytesToRead.map { max(0, $0 - bytesRead) } if remaining == 0 { if let fileSize, startOffset + bytesRead >= fileSize, hasCompleteJSONTail() { - flushLine() + flushLine(endOffset: startOffset + bytesRead) committedOffset = startOffset + bytesRead lineStartOffset = committedOffset } @@ -368,7 +391,7 @@ enum CostUsageJsonl { let chunk = try handle.read(upToCount: readCount) ?? Data() if chunk.isEmpty { if hasCompleteJSONTail() { - flushLine() + flushLine(endOffset: startOffset + bytesRead) committedOffset = startOffset + bytesRead lineStartOffset = committedOffset } @@ -385,8 +408,9 @@ enum CostUsageJsonl { while index < rawBuffer.count { if base[index] == 0x0A { appendSegment(base.advanced(by: segmentStart), count: index - segmentStart) - flushLine() - committedOffset = chunkStartOffset + Int64(index + 1) + let lineEndOffset = chunkStartOffset + Int64(index + 1) + flushLine(endOffset: lineEndOffset) + committedOffset = lineEndOffset lineStartOffset = committedOffset segmentStart = index + 1 } else { diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift index d6c5882db9..bb8736d9cd 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift @@ -302,12 +302,17 @@ extension CostUsageScanner { codexPriorityTokens: [String: [String: Int]]? = nil, codexTurnIDs: [String]? = nil, codexRows: [CodexUsageRow]? = nil, + codexTokenSnapshots: [CostUsageCodexTokenSnapshot]? = nil, + codexTokenCheckpoints: [CostUsageCodexTokenCheckpoint]? = nil, + codexTokenTimestampsMonotonic: Bool? = nil, + codexTokenIndexAnchor: CostUsageCodexTokenIndexAnchor? = nil, claudeRows: [ClaudeUsageRow]? = nil, codexScanFileId: String? = nil, codexScanTargetSize: Int64? = nil, codexScanComplete: Bool? = nil, codexJSONLResumeState: CostUsageJsonl.ResumeState? = nil, - codexBufferedSubagentLines: [CodexBufferedFastLine]? = nil) -> CostUsageFileUsage + codexBufferedSubagentLines: [CodexBufferedFastLine]? = nil, + codexBufferedUnresolvedForkLines: [CodexBufferedFastLine]? = nil) -> CostUsageFileUsage { CostUsageFileUsage( mtimeUnixMs: mtimeUnixMs, @@ -338,12 +343,17 @@ extension CostUsageScanner { codexPriorityTokens: codexPriorityTokens, codexTurnIDs: codexTurnIDs, codexRows: codexRows, + codexTokenSnapshots: codexTokenSnapshots, + codexTokenCheckpoints: codexTokenCheckpoints, + codexTokenTimestampsMonotonic: codexTokenTimestampsMonotonic, + codexTokenIndexAnchor: codexTokenIndexAnchor, claudeRows: claudeRows, codexScanFileId: codexScanFileId, codexScanTargetSize: codexScanTargetSize, codexScanComplete: codexScanComplete, codexJSONLResumeState: codexJSONLResumeState, - codexBufferedSubagentLines: codexBufferedSubagentLines) + codexBufferedSubagentLines: codexBufferedSubagentLines, + codexBufferedUnresolvedForkLines: codexBufferedUnresolvedForkLines) } static func needsCodexCostCache(_ usage: CostUsageFileUsage) -> Bool { @@ -805,7 +815,17 @@ extension CostUsageScanner { Self.intMapOutsideScanWindow(usage.codexPriorityTokens, range: context.range), splitMaps.priorityTokens), codexTurnIDs: Self.mergeCodexTurnIDs(nil, rows: rows), - codexRows: rows) + codexRows: rows, + codexTokenSnapshots: usage.codexTokenSnapshots, + codexTokenCheckpoints: usage.codexTokenCheckpoints, + codexTokenTimestampsMonotonic: usage.codexTokenTimestampsMonotonic, + codexTokenIndexAnchor: usage.codexTokenIndexAnchor, + codexScanFileId: usage.codexScanFileId, + codexScanTargetSize: usage.codexScanTargetSize, + codexScanComplete: usage.codexScanComplete, + codexJSONLResumeState: usage.codexJSONLResumeState, + codexBufferedSubagentLines: usage.codexBufferedSubagentLines, + codexBufferedUnresolvedForkLines: usage.codexBufferedUnresolvedForkLines) .refreshingCodexWorkspaceUsageFingerprint() } @@ -975,8 +995,9 @@ extension CostUsageScanner { if let parentSessionId = cached.forkedFromId { guard let cachedDependencyKey = cached.forkBaselineDependencyKey else { return false } if cachedDependencyKey != Self.codexForkDependencyNotRequiredKey { - let currentDependencyKey = try context.resources.inheritedResolver + guard let currentDependencyKey = try context.resources.inheritedResolver .currentDependencyKey(for: parentSessionId) + else { return false } guard cachedDependencyKey == currentDependencyKey else { return false } } } @@ -1053,13 +1074,25 @@ extension CostUsageScanner { let isResumablePartial = cached.codexScanComplete == false && cached.codexScanFileId != nil && cached.codexScanFileId == input.metadata.fileId - && cached.codexScanTargetSize == input.metadata.size - && cached.mtimeUnixMs == input.metadata.mtimeUnixMs + && startOffset > 0 + && startOffset <= input.metadata.size + && cached.codexTokenIndexAnchor?.indexedBytes == startOffset + && cached.codexTokenIndexAnchor.map { + CostUsageScanner.codexTokenIndexAnchorMatches( + $0, + fileURL: input.fileURL, + metadata: input.metadata) + } == true && hasMatchingResumeOffset + let isBufferedForkRetry = cached.forkedFromId != nil + && cached.forkBaselineDependencyKey == nil + && cached.hasBufferedCodexForkRetryLines + && cached.codexScanFileId == input.metadata.fileId + && startOffset == input.metadata.size if cached.codexScanComplete == false, !isResumablePartial { return false } - if !isResumablePartial, try Self.codexFileIsSubagentThread( + if !isResumablePartial, !isBufferedForkRetry, try Self.codexFileIsSubagentThread( fileURL: input.fileURL, checkCancellation: context.checkCancellation) { @@ -1077,6 +1110,7 @@ extension CostUsageScanner { let canIncremental = startOffset > 0 && startOffset <= input.metadata.size && (isResumablePartial + || isBufferedForkRetry || (input.metadata.size > cached.size && initialCountedTotals != nil && cached.forkedFromId == nil @@ -1097,10 +1131,15 @@ extension CostUsageScanner { initialCodexTurnID: cached.lastCodexTurnID, initialCodexUsageRowIndex: Self.nextCodexUsageRowIndex(cached.codexRows), initialBufferedSubagentLines: cached.codexBufferedSubagentLines, + initialBufferedUnresolvedForkLines: cached.codexBufferedUnresolvedForkLines, initialJSONLResumeState: cached.codexJSONLResumeState, maxBytesToRead: maxBytesToRead, + shouldStopReading: context.scanBudget.map { budget in + { bytesRead in budget.shouldYield(additionalBytes: bytesRead) } + }, + inheritedTotalsResolver: context.resources.inheritedResolver.inheritedTotals(for:atOrBefore:), checkCancellation: context.checkCancellation) - if delta.forkedFromId != nil, !isResumablePartial { + if delta.forkedFromId != nil, !isResumablePartial, !isBufferedForkRetry { return false } let migrated = Self.codexFileUsageWithCostCache(cached, context: context) @@ -1169,6 +1208,9 @@ extension CostUsageScanner { priorityTurns: context.resources.priorityTurns, modelsDevCatalog: context.resources.modelsDevCatalog, modelsDevCacheRoot: context.resources.modelsDevCacheRoot) + let mergedTokenSnapshots = isBufferedForkRetry + ? (migratedCached.codexTokenSnapshots ?? []) + : (migratedCached.codexTokenSnapshots ?? []) + delta.tokenSnapshots cache.files[input.metadata.path] = Self.makeFileUsage( mtimeUnixMs: input.metadata.mtimeUnixMs, size: input.metadata.size, @@ -1215,11 +1257,18 @@ extension CostUsageScanner { priorityTurns: context.resources.priorityTurns, modelsDevCatalog: context.resources.modelsDevCatalog, modelsDevCacheRoot: context.resources.modelsDevCacheRoot), + codexTokenSnapshots: mergedTokenSnapshots, + codexTokenCheckpoints: Self.codexTokenCheckpoints(for: mergedTokenSnapshots), + codexTokenTimestampsMonotonic: Self.codexTokenTimestampsAreMonotonic(mergedTokenSnapshots), + codexTokenIndexAnchor: Self.codexTokenIndexAnchor( + fileURL: input.fileURL, + indexedBytes: delta.parsedBytes), codexScanFileId: input.metadata.fileId, codexScanTargetSize: input.metadata.size, codexScanComplete: delta.parsedBytes >= input.metadata.size && delta.jsonlResumeState == nil, codexJSONLResumeState: delta.jsonlResumeState, - codexBufferedSubagentLines: delta.bufferedSubagentLines) + codexBufferedSubagentLines: delta.bufferedSubagentLines, + codexBufferedUnresolvedForkLines: delta.bufferedUnresolvedForkLines) .refreshingCodexWorkspaceUsageFingerprint() Self.rememberScannedCodexFile( input: input, @@ -1250,6 +1299,9 @@ extension CostUsageScanner { fileURL: input.fileURL, range: context.range, maxBytesToRead: maxBytesToRead, + shouldStopReading: context.scanBudget.map { budget in + { bytesRead in budget.shouldYield(additionalBytes: bytesRead) } + }, inheritedTotalsResolver: context.resources.inheritedResolver.inheritedTotals(for:atOrBefore:), checkCancellation: context.checkCancellation) let forkBaselineDependencyKey = Self.codexForkBaselineDependencyKey( @@ -1362,11 +1414,18 @@ extension CostUsageScanner { priorityTurns: context.resources.priorityTurns, modelsDevCatalog: context.resources.modelsDevCatalog, modelsDevCacheRoot: context.resources.modelsDevCacheRoot), + codexTokenSnapshots: parsed.tokenSnapshots, + codexTokenCheckpoints: Self.codexTokenCheckpoints(for: parsed.tokenSnapshots), + codexTokenTimestampsMonotonic: Self.codexTokenTimestampsAreMonotonic(parsed.tokenSnapshots), + codexTokenIndexAnchor: Self.codexTokenIndexAnchor( + fileURL: input.fileURL, + indexedBytes: parsed.parsedBytes), codexScanFileId: input.metadata.fileId, codexScanTargetSize: input.metadata.size, codexScanComplete: parsed.parsedBytes >= input.metadata.size && parsed.jsonlResumeState == nil, codexJSONLResumeState: parsed.jsonlResumeState, - codexBufferedSubagentLines: parsed.bufferedSubagentLines) + codexBufferedSubagentLines: parsed.bufferedSubagentLines, + codexBufferedUnresolvedForkLines: parsed.bufferedUnresolvedForkLines) .refreshingCodexWorkspaceUsageFingerprint() Self.applyFileDays(cache: &cache, fileDays: cache.files[input.metadata.path]?.days ?? [:], sign: 1) Self.rememberScannedCodexFile( diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index fe2677734a..cdcad4999d 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -18,6 +18,25 @@ enum CostUsageScanner { /// this value records that lineage exists but this rollout owns its counter or suffix. static let codexForkDependencyNotRequiredKey = "mode:lineage-only:v1" + final class CodexSessionHeadParseObserverStore: @unchecked Sendable { + let observer: () -> Void + + init(observer: @escaping () -> Void) { + self.observer = observer + } + } + + @TaskLocal private static var codexSessionHeadParseObserverStore: CodexSessionHeadParseObserverStore? + + static func withCodexSessionHeadParseObserverForTesting( + _ observer: @escaping () -> Void, + operation: () throws -> T) rethrows -> T + { + try self.$codexSessionHeadParseObserverStore.withValue(.init(observer: observer)) { + try operation() + } + } + enum ClaudeLogProviderFilter { case all case vertexAIOnly @@ -40,6 +59,9 @@ enum CostUsageScanner { /// Soft budget for newly-read Codex session bytes in one refresh. /// Remaining dirty files are deferred to later refreshes. Default 512 MiB. var maxCodexScanBytesPerRefresh: Int64 = 512 * 1024 * 1024 + /// Optional wall-clock budget for newly-read Codex bytes in one refresh. The reader + /// finishes its current 256 KiB chunk, persists resume state, and continues later. + var maxCodexScanDurationPerRefresh: TimeInterval? /// Prefer newest session files first so recent usage lands before catch-up work. var preferNewestCodexSessionsFirst: Bool = true @@ -53,6 +75,7 @@ enum CostUsageScanner { forceRescan: Bool = false, maxCodexSessionFileBytes: Int64 = 256 * 1024 * 1024, maxCodexScanBytesPerRefresh: Int64 = 512 * 1024 * 1024, + maxCodexScanDurationPerRefresh: TimeInterval? = nil, preferNewestCodexSessionsFirst: Bool = true) { self.codexSessionsRoot = codexSessionsRoot @@ -64,6 +87,7 @@ enum CostUsageScanner { self.forceRescan = forceRescan self.maxCodexSessionFileBytes = max(0, maxCodexSessionFileBytes) self.maxCodexScanBytesPerRefresh = max(0, maxCodexScanBytesPerRefresh) + self.maxCodexScanDurationPerRefresh = maxCodexScanDurationPerRefresh.map { max(0, $0) } self.preferNewestCodexSessionsFirst = preferNewestCodexSessionsFirst } } @@ -76,10 +100,30 @@ enum CostUsageScanner { private(set) var bytesConsumed: Int64 = 0 private(set) var resumedPartialFileCount = 0 private(set) var deferredByBudgetFileCount = 0 + private(set) var deferredByTimeBudgetFileCount = 0 + private var bytesReserved: Int64 = 0 + private let deadline: ContinuousClock.Instant? + private let now: @Sendable () -> ContinuousClock.Instant + private var recordedTimeDeferral = false - init(maxFileBytes: Int64, maxBytesPerRefresh: Int64) { + init( + maxFileBytes: Int64, + maxBytesPerRefresh: Int64, + maxDuration: TimeInterval? = nil, + now: @escaping @Sendable () -> ContinuousClock.Instant = { ContinuousClock.now }) + { self.maxFileBytes = max(0, maxFileBytes) self.maxBytesPerRefresh = max(0, maxBytesPerRefresh) + self.now = now + if let maxDuration, maxDuration > 0 { + self.deadline = now().advanced(by: .seconds(maxDuration)) + } else { + self.deadline = nil + } + } + + var hasTimeLimit: Bool { + self.deadline != nil } enum Admission { @@ -89,8 +133,12 @@ enum CostUsageScanner { func admit(workBytes: Int64) -> Admission { let work = max(0, workBytes) + if work > 0, self.shouldYield(additionalBytes: 0) { + self.deferredByBudgetFileCount += 1 + return .deferBudget + } let refreshRemaining = self.maxBytesPerRefresh > 0 - ? max(0, self.maxBytesPerRefresh - self.bytesConsumed) + ? max(0, self.maxBytesPerRefresh - self.bytesConsumed - self.bytesReserved) : Int64.max if work > 0, refreshRemaining == 0 { self.deferredByBudgetFileCount += 1 @@ -101,11 +149,36 @@ enum CostUsageScanner { if allowance < work { self.resumedPartialFileCount += 1 } + self.bytesReserved += allowance return .allow(allowance) } func consume(workBytes: Int64) { - self.bytesConsumed += max(0, workBytes) + let work = max(0, workBytes) + self.bytesReserved = max(0, self.bytesReserved - work) + self.bytesConsumed += work + } + + func release(workBytes: Int64) { + self.bytesReserved = max(0, self.bytesReserved - max(0, workBytes)) + } + + func complete(admittedWorkBytes: Int64, actualWorkBytes: Int64) { + let admitted = max(0, admittedWorkBytes) + let actual = min(admitted, max(0, actualWorkBytes)) + self.consume(workBytes: actual) + self.release(workBytes: admitted - actual) + } + + func shouldYield(additionalBytes: Int64) -> Bool { + guard let deadline else { return false } + guard self.bytesConsumed + self.bytesReserved + max(0, additionalBytes) > 0 else { return false } + guard self.now() >= deadline else { return false } + if !self.recordedTimeDeferral { + self.recordedTimeDeferral = true + self.deferredByTimeBudgetFileCount += 1 + } + return true } } @@ -127,8 +200,10 @@ enum CostUsageScanner { let projectPath: String? let codexSession: CostUsageCodexSessionMetadata let rows: [CodexUsageRow] + let tokenSnapshots: [CostUsageCodexTokenSnapshot] let jsonlResumeState: CostUsageJsonl.ResumeState? let bufferedSubagentLines: [CodexBufferedFastLine]? + let bufferedUnresolvedForkLines: [CodexBufferedFastLine]? } struct CodexUsageRow: Codable, Equatable { @@ -363,7 +438,9 @@ enum CostUsageScanner { private static func codexOptionalDelta(from baseline: Int?, to current: Int?, hasBaseline: Bool) -> Int? { guard let current else { return nil } - if !hasBaseline { return current } + if !hasBaseline { + return current + } guard let baseline else { return nil } return max(0, current - baseline) } @@ -465,12 +542,33 @@ enum CostUsageScanner { /// Cumulative-totals accounting for parent-session snapshot building. Applies the same /// containment policy as `parseCodexFileCancellable` so fork children inherit baselines /// computed under identical rules. - private struct CodexSnapshotAccumulator { + struct CodexSnapshotAccumulator { var countedTotals: CostUsageCodexTotals? var rawTotalsBaseline: CostUsageCodexTotals? var sawDivergentTotals = false var tracker = CodexTotalsTracker() + init(state: CostUsageCodexTokenAccumulatorState? = nil) { + guard let state else { return } + self.countedTotals = state.countedTotals + self.rawTotalsBaseline = state.rawTotalsBaseline + self.sawDivergentTotals = state.sawDivergentTotals + self.tracker = CodexTotalsTracker( + watermark: state.rawTotalsWatermark, + seenRawTotals: state.seenRawTotals, + sawInterleavedTotals: state.sawInterleavedTotals) + } + + var state: CostUsageCodexTokenAccumulatorState { + CostUsageCodexTokenAccumulatorState( + countedTotals: self.countedTotals, + rawTotalsBaseline: self.rawTotalsBaseline, + sawDivergentTotals: self.sawDivergentTotals, + rawTotalsWatermark: self.tracker.watermark, + seenRawTotals: self.tracker.seenRawTotals, + sawInterleavedTotals: self.tracker.sawInterleavedTotals) + } + /// Applies one token-count event and returns the counted cumulative totals afterwards. mutating func apply( last: CostUsageCodexTotals?, @@ -559,6 +657,52 @@ enum CostUsageScanner { } } + static let codexTokenCheckpointStride: Int64 = 4 * 1024 * 1024 + + static func codexTokenCheckpoints( + for events: [CostUsageCodexTokenSnapshot]) -> [CostUsageCodexTokenCheckpoint] + { + guard !events.isEmpty else { return [] } + var accumulator = CodexSnapshotAccumulator() + var checkpoints: [CostUsageCodexTokenCheckpoint] = [] + var lastCheckpointOffset: Int64 = 0 + + for (eventIndex, event) in events.enumerated() { + _ = accumulator.apply(last: event.last, total: event.total) + guard let endOffset = event.endOffset else { continue } + let reachedStride = endOffset - lastCheckpointOffset >= Self.codexTokenCheckpointStride + let isLastEvent = eventIndex == events.index(before: events.endIndex) + guard reachedStride || isLastEvent else { continue } + checkpoints.append(CostUsageCodexTokenCheckpoint( + eventIndex: eventIndex, + timestamp: event.timestamp, + endOffset: endOffset, + state: accumulator.state)) + lastCheckpointOffset = endOffset + } + + return checkpoints + } + + static func codexTokenTimestampsAreMonotonic( + _ events: [CostUsageCodexTokenSnapshot]) -> Bool + { + guard events.count > 1 else { return true } + for (previous, current) in zip(events, events.dropFirst()) { + let isOrdered: Bool = if let previousDate = Self.dateFromTimestamp(previous.timestamp), + let currentDate = Self.dateFromTimestamp(current.timestamp) + { + previousDate <= currentDate + } else { + previous.timestamp <= current.timestamp + } + if !isOrdered { + return false + } + } + return true + } + struct CodexScanResources { let fileIndex: CodexSessionFileIndex let inheritedResolver: CodexInheritedTotalsResolver @@ -691,114 +835,537 @@ enum CostUsageScanner { } final class CodexSessionFileIndex { + enum Lookup { + case found(URL) + case missing(dependencyKey: String) + case deferred + } + + private enum InventoryValidation { + case current + case changed + case deferred + } + private let files: [URL] - private let filePaths: Set private let roots: [URL] private let checkCancellation: CancellationCheck? - private var nextUnindexedFile = 0 - private var didIndexRoots = false - private var fileURLBySessionId: [String: URL] = [:] - private var missingSessionIds: Set = [] + private let scanBudget: CodexScanBudget? + private let headParseObserver: (() -> Void)? + private var discovery: CostUsageCodexSessionDiscovery init( files: [URL], roots: [URL], cachedSessionFiles: [String: URL] = [:], + cachedDiscovery: CostUsageCodexSessionDiscovery? = nil, + scanBudget: CodexScanBudget? = nil, + headParseObserver: (() -> Void)? = nil, checkCancellation: CancellationCheck? = nil) { self.files = files - self.filePaths = Set(files.map(\.path)) self.roots = roots - self.fileURLBySessionId = cachedSessionFiles self.checkCancellation = checkCancellation + self.scanBudget = scanBudget + self.headParseObserver = headParseObserver + let rootPaths = roots.map(\.standardizedFileURL.path).sorted() + if var cachedDiscovery, cachedDiscovery.roots == rootPaths { + for (sessionId, fileURL) in cachedSessionFiles { + cachedDiscovery.filePathBySessionId[sessionId] = fileURL.standardizedFileURL.path + } + self.discovery = cachedDiscovery + if !cachedDiscovery.isComplete { + self.enqueueCurrentFiles() + } + } else { + self.discovery = Self.makeFreshDiscovery( + roots: roots, + files: files, + cachedSessionFiles: cachedSessionFiles, + retaining: nil) + } + } + + var persistedState: CostUsageCodexSessionDiscovery { + self.discovery + } + + var hasPendingDiscovery: Bool { + !self.discovery.isComplete + && (!self.discovery.pendingSessionIds.isEmpty || self.discovery.headScan != nil) } func remember(fileURL: URL, sessionId: String?) { guard let sessionId, !sessionId.isEmpty else { return } - self.fileURLBySessionId[sessionId] = fileURL + let path = fileURL.standardizedFileURL.path + self.discovery.filePathBySessionId[sessionId] = path + self.discovery.missingSessionIds.removeAll { $0 == sessionId } + self.discovery.pendingSessionIds.removeAll { $0 == sessionId } + self.discovery.fileStamps[path] = Self.fileStamp(fileURL: fileURL) } - func fileURL(for sessionId: String) throws -> URL? { - if let cached = self.fileURLBySessionId[sessionId] { - return cached + func lookup(sessionId: String) throws -> Lookup { + if let cached = self.cachedFileURL(for: sessionId) { + return .found(cached) } - if self.missingSessionIds.contains(sessionId) { + + if self.discovery.isComplete { + switch try self.validateInventory() { + case .current: + if self.discovery.missingSessionIds.contains(sessionId), + let generation = self.discovery.generation + { + return .missing(dependencyKey: Self.missingDependencyKey( + sessionId: sessionId, + generation: generation)) + } + case .changed: + self.discovery = Self.makeFreshDiscovery( + roots: self.roots, + files: self.files, + cachedSessionFiles: self.cachedSessionFiles(), + retaining: self.discovery) + case .deferred: + return .deferred + } + } + + if !self.discovery.pendingSessionIds.contains(sessionId) { + self.discovery.pendingSessionIds.append(sessionId) + } + return try self.resumeDiscovery(requestedSessionId: sessionId) + } + + private func cachedFileURL(for sessionId: String) -> URL? { + guard let path = self.discovery.filePathBySessionId[sessionId] else { return nil } + guard FileManager.default.fileExists(atPath: path) else { + self.discovery.filePathBySessionId.removeValue(forKey: sessionId) return nil } + return URL(fileURLWithPath: path) + } + + private func cachedSessionFiles() -> [String: URL] { + self.discovery.filePathBySessionId.reduce(into: [:]) { result, entry in + guard FileManager.default.fileExists(atPath: entry.value) else { return } + result[entry.key] = URL(fileURLWithPath: entry.value) + } + } - while self.nextUnindexedFile < self.files.count { + private func resumeDiscovery(requestedSessionId: String) throws -> Lookup { + while true { try self.checkCancellation?() - let fileURL = self.files[self.nextUnindexedFile] - self.nextUnindexedFile += 1 - guard let indexedSessionId = try CostUsageScanner.parseCodexSessionIdentifier( - fileURL: fileURL, - checkCancellation: self.checkCancellation) - else { + if let cached = self.cachedFileURL(for: requestedSessionId) { + return .found(cached) + } + + if self.discovery.nextFileIndex < self.discovery.filePaths.count { + guard try self.scanNextFileHead() else { return .deferred } continue } - self.fileURLBySessionId[indexedSessionId] = fileURL - if indexedSessionId == sessionId { - return fileURL + + if self.discovery.nextDirectoryIndex < self.discovery.directoryPaths.count { + guard try self.enumerateNextDirectory() else { return .deferred } + continue + } + + self.finishDiscovery() + if let cached = self.cachedFileURL(for: requestedSessionId) { + return .found(cached) } + let generation = self.discovery.generation ?? "unknown" + return .missing(dependencyKey: Self.missingDependencyKey( + sessionId: requestedSessionId, + generation: generation)) } + } - if !self.didIndexRoots { - try self.indexRoots() - if let indexed = self.fileURLBySessionId[sessionId] { - return indexed + private func scanNextFileHead() throws -> Bool { + let path = self.discovery.filePaths[self.discovery.nextFileIndex] + let fileURL = URL(fileURLWithPath: path) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + guard metadata.fileId != nil else { + self.advancePastHead(path: path, stamp: nil) + return true + } + + var head = self.discovery.headScan + if head?.path != path { + head = CostUsageCodexSessionDiscovery.HeadScan(path: path, offset: 0, resumeState: nil) + } + let startOffset = head?.resumeState?.offset ?? head?.offset ?? 0 + let remainingBytes = max(0, metadata.size - startOffset) + let admittedBytes: Int64 + if let scanBudget = self.scanBudget { + switch scanBudget.admit(workBytes: remainingBytes) { + case let .allow(allowance): admittedBytes = allowance + case .deferBudget: return false } + } else { + admittedBytes = remainingBytes } - self.missingSessionIds.insert(sessionId) - return nil + self.headParseObserver?() + let result = try CostUsageScanner.scanCodexSessionIdentifier( + fileURL: fileURL, + offset: head?.offset ?? 0, + maxBytesToRead: admittedBytes, + resumeState: head?.resumeState, + checkCancellation: self.checkCancellation) + self.scanBudget?.complete( + admittedWorkBytes: admittedBytes, + actualWorkBytes: result.bytesRead) + + if let sessionId = result.sessionId, !sessionId.isEmpty { + self.discovery.filePathBySessionId[sessionId] = path + self.advancePastHead(path: path, stamp: Self.fileStamp(metadata: metadata)) + return true + } + if result.isComplete { + self.advancePastHead(path: path, stamp: Self.fileStamp(metadata: metadata)) + return true + } + + self.discovery.headScan = CostUsageCodexSessionDiscovery.HeadScan( + path: path, + offset: result.committedOffset, + resumeState: result.resumeState) + return false } - private func indexRoots() throws { - self.didIndexRoots = true - guard !self.roots.isEmpty else { return } - for root in self.roots { + private func advancePastHead( + path: String, + stamp: CostUsageCodexSessionDiscovery.FileStamp?) + { + if let stamp { + self.discovery.fileStamps[path] = stamp + } else { + self.discovery.fileStamps.removeValue(forKey: path) + self.discovery.filePathBySessionId = self.discovery.filePathBySessionId.filter { $0.value != path } + } + self.discovery.headScan = nil + self.discovery.nextFileIndex += 1 + } + + private func enumerateNextDirectory() throws -> Bool { + let admittedWork: Int64 + if let scanBudget = self.scanBudget { + switch scanBudget.admit(workBytes: 1) { + case let .allow(allowance): admittedWork = allowance + case .deferBudget: return false + } + } else { + admittedWork = 1 + } + defer { + self.scanBudget?.complete(admittedWorkBytes: admittedWork, actualWorkBytes: admittedWork) + } + + try self.checkCancellation?() + let path = self.discovery.directoryPaths[self.discovery.nextDirectoryIndex] + let directoryURL = URL(fileURLWithPath: path, isDirectory: true) + let items = (try? FileManager.default.contentsOfDirectory( + at: directoryURL, + includingPropertiesForKeys: [.isDirectoryKey, .isRegularFileKey], + options: [.skipsHiddenFiles, .skipsPackageDescendants])) ?? [] + var jsonlFileCount = 0 + for item in items { try self.checkCancellation?() - guard let enumerator = FileManager.default.enumerator( - at: root, - includingPropertiesForKeys: [.isRegularFileKey], - options: [.skipsHiddenFiles, .skipsPackageDescendants]) - else { continue } - - while let fileURL = enumerator.nextObject() as? URL { - try self.checkCancellation?() - guard fileURL.pathExtension.lowercased() == "jsonl" else { continue } - guard !self.filePaths.contains(fileURL.path) else { continue } - guard let indexedSessionId = try CostUsageScanner.parseCodexSessionIdentifier( - fileURL: fileURL, - checkCancellation: self.checkCancellation) - else { - continue + let values = try? item.resourceValues(forKeys: [.isDirectoryKey, .isRegularFileKey]) + if values?.isDirectory == true { + self.enqueueDirectory(item) + } else if item.pathExtension.lowercased() == "jsonl" { + jsonlFileCount += 1 + self.enqueueFile(item) + } + } + let metadata = CostUsageScanner.codexFileMetadata(fileURL: directoryURL) + self.discovery.directoryStamps[path] = .init( + mtimeUnixMs: metadata.mtimeUnixMs, + jsonlFileCount: jsonlFileCount) + self.discovery.nextDirectoryIndex += 1 + return !self.scanBudgetExhausted() + } + + private func enqueueCurrentFiles() { + for fileURL in self.files { + self.enqueueFile(fileURL) + } + } + + private func enqueueFile(_ fileURL: URL) { + let path = fileURL.standardizedFileURL.path + guard !self.discovery.filePaths.contains(path) else { return } + self.discovery.filePaths.append(path) + } + + private func enqueueDirectory(_ directoryURL: URL) { + let path = directoryURL.standardizedFileURL.path + guard !self.discovery.directoryPaths.contains(path) else { return } + self.discovery.directoryPaths.append(path) + } + + private func finishDiscovery() { + let generation = Self.discoveryGeneration( + roots: self.discovery.roots, + directoryStamps: self.discovery.directoryStamps) + self.discovery.generation = generation + for sessionId in self.discovery.pendingSessionIds + where self.discovery.filePathBySessionId[sessionId] == nil + { + if !self.discovery.missingSessionIds.contains(sessionId) { + self.discovery.missingSessionIds.append(sessionId) + } + } + self.discovery.missingSessionIds.sort() + self.discovery.pendingSessionIds.removeAll() + self.discovery.directoryPaths = self.discovery.directoryStamps.keys.sorted() + self.discovery.nextDirectoryIndex = self.discovery.directoryPaths.count + self.discovery.validationDirectoryIndex = 0 + self.discovery.isComplete = true + } + + private func validateInventory() throws -> InventoryValidation { + while self.discovery.validationDirectoryIndex < self.discovery.directoryPaths.count { + let admittedWork: Int64 + if let scanBudget = self.scanBudget { + switch scanBudget.admit(workBytes: 1) { + case let .allow(allowance): admittedWork = allowance + case .deferBudget: return .deferred } - self.fileURLBySessionId[indexedSessionId] = fileURL + } else { + admittedWork = 1 + } + + try self.checkCancellation?() + let path = self.discovery.directoryPaths[self.discovery.validationDirectoryIndex] + let currentMtime = Self.directoryModificationTime(atPath: path) + self.scanBudget?.complete(admittedWorkBytes: admittedWork, actualWorkBytes: admittedWork) + guard currentMtime == self.discovery.directoryStamps[path]?.mtimeUnixMs else { + self.discovery.validationDirectoryIndex = 0 + return .changed + } + self.discovery.validationDirectoryIndex += 1 + if self.scanBudgetExhausted() { + return .deferred } } + self.discovery.validationDirectoryIndex = 0 + return .current } + + private func scanBudgetExhausted() -> Bool { + guard let scanBudget = self.scanBudget else { return false } + switch scanBudget.admit(workBytes: 1) { + case let .allow(allowance): + scanBudget.release(workBytes: allowance) + return false + case .deferBudget: + return true + } + } + + private static func makeFreshDiscovery( + roots: [URL], + files: [URL], + cachedSessionFiles: [String: URL], + retaining previous: CostUsageCodexSessionDiscovery?) -> CostUsageCodexSessionDiscovery + { + let rootPaths = roots.map(\.standardizedFileURL.path).sorted() + var retainedStamps: [String: CostUsageCodexSessionDiscovery.FileStamp] = [:] + if let previous { + for (path, stamp) in previous.fileStamps { + let current = Self.fileStamp(fileURL: URL(fileURLWithPath: path)) + if current == stamp { + retainedStamps[path] = stamp + } + } + } + for fileURL in cachedSessionFiles.values { + let path = fileURL.standardizedFileURL.path + if let stamp = Self.fileStamp(fileURL: fileURL) { + retainedStamps[path] = stamp + } + } + + let retainedPaths = retainedStamps.keys.sorted() + var sessionFiles = previous?.filePathBySessionId.filter { + retainedStamps[$0.value] != nil + } ?? [:] + for (sessionId, fileURL) in cachedSessionFiles { + sessionFiles[sessionId] = fileURL.standardizedFileURL.path + } + var filePaths = retainedPaths + var knownPaths = Set(filePaths) + for fileURL in files { + let path = fileURL.standardizedFileURL.path + if knownPaths.insert(path).inserted { + filePaths.append(path) + } + } + return CostUsageCodexSessionDiscovery( + roots: rootPaths, + generation: nil, + directoryStamps: [:], + directoryPaths: rootPaths, + nextDirectoryIndex: 0, + filePaths: filePaths, + nextFileIndex: retainedPaths.count, + fileStamps: retainedStamps, + headScan: nil, + filePathBySessionId: sessionFiles, + missingSessionIds: [], + pendingSessionIds: [], + validationDirectoryIndex: 0, + isComplete: false) + } + + private static func directoryModificationTime(atPath path: String) -> Int64? { + let url = URL(fileURLWithPath: path, isDirectory: true) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: url) + guard metadata.fileId != nil else { return nil } + return metadata.mtimeUnixMs + } + + private static func fileStamp( + fileURL: URL) -> CostUsageCodexSessionDiscovery.FileStamp? + { + self.fileStamp(metadata: CostUsageScanner.codexFileMetadata(fileURL: fileURL)) + } + + private static func fileStamp( + metadata: CodexFileMetadata) -> CostUsageCodexSessionDiscovery.FileStamp? + { + guard metadata.fileId != nil else { return nil } + return .init(mtimeUnixMs: metadata.mtimeUnixMs, size: metadata.size, fileId: metadata.fileId) + } + + private static func discoveryGeneration( + roots: [String], + directoryStamps: [String: CostUsageCodexSessionDiscovery.DirectoryStamp]) -> String + { + let directories = directoryStamps.map { path, stamp in + "\(path)|\(stamp.mtimeUnixMs)|\(stamp.jsonlFileCount)" + }.sorted() + return CostUsageScanner.sha256Hex(Data((roots + directories).joined(separator: "\n").utf8)) + } + + private static func missingDependencyKey(sessionId: String, generation: String) -> String { + "missing|\(sessionId)|discovery|\(generation)" + } + } + + private struct CodexSessionIdentifierScanResult { + let sessionId: String? + let bytesRead: Int64 + let committedOffset: Int64 + let resumeState: CostUsageJsonl.ResumeState? + let isComplete: Bool + } + + private static func scanCodexSessionIdentifier( + fileURL: URL, + offset: Int64, + maxBytesToRead: Int64, + resumeState: CostUsageJsonl.ResumeState?, + checkCancellation: CancellationCheck?) throws -> CodexSessionIdentifierScanResult + { + var sessionId: String? + let scanStart = resumeState?.offset ?? max(0, offset) + let progress = try CostUsageJsonl.scanBounded( + fileURL: fileURL, + offset: offset, + maxLineBytes: Self.codexSessionMetadataMaxLineBytes, + prefixBytes: Self.codexSessionMetadataMaxLineBytes, + maxBytesToRead: maxBytesToRead, + resumeState: resumeState, + shouldStop: { _ in sessionId != nil }, + checkCancellation: checkCancellation, + onLine: { line in + guard !line.wasTruncated else { return } + if case let .sessionMeta(metadata) = Self.parseCodexFastLine(line.bytes) { + sessionId = metadata.sessionId + } + }) + let size = Self.codexFileMetadata(fileURL: fileURL).size + return CodexSessionIdentifierScanResult( + sessionId: sessionId, + bytesRead: max(0, progress.readOffset - scanStart), + committedOffset: progress.committedOffset, + resumeState: progress.resumeState, + isComplete: sessionId != nil || progress.readOffset >= size) } final class CodexInheritedTotalsResolver { private struct SnapshotResolution { let dependencyKey: String? let snapshots: [CodexTimestampedTotals]? + let indexedEvents: [CostUsageCodexTokenSnapshot]? + let checkpoints: [CostUsageCodexTokenCheckpoint] + let indexedTimestampsMonotonic: Bool + let isComplete: Bool + + init( + dependencyKey: String?, + snapshots: [CodexTimestampedTotals]? = nil, + indexedEvents: [CostUsageCodexTokenSnapshot]? = nil, + checkpoints: [CostUsageCodexTokenCheckpoint] = [], + indexedTimestampsMonotonic: Bool = false, + isComplete: Bool) + { + self.dependencyKey = dependencyKey + self.snapshots = snapshots + self.indexedEvents = indexedEvents + self.checkpoints = checkpoints + self.indexedTimestampsMonotonic = indexedTimestampsMonotonic + self.isComplete = isComplete + } + + var lastTimestamp: String? { + self.indexedEvents?.last?.timestamp ?? self.snapshots?.last?.timestamp + } + + var hasSnapshotSource: Bool { + self.indexedEvents != nil || self.snapshots != nil + } } private let fileIndex: CodexSessionFileIndex private let checkCancellation: CancellationCheck? private let scanBudget: CodexScanBudget? + private var cachedFiles: [String: CostUsageFileUsage] private var snapshotResolutions: [String: SnapshotResolution] = [:] + private var resolvedDependencyKeys: [String: String] = [:] + private var pendingParentFiles: [String: URL] = [:] init( fileIndex: CodexSessionFileIndex, checkCancellation: CancellationCheck?, - scanBudget: CodexScanBudget? = nil) + scanBudget: CodexScanBudget? = nil, + cachedFiles: [String: CostUsageFileUsage] = [:]) { self.fileIndex = fileIndex self.checkCancellation = checkCancellation self.scanBudget = scanBudget + self.cachedFiles = cachedFiles + } + + func updateCachedUsage(fileURL: URL, usage: CostUsageFileUsage?) { + let path = fileURL.path + let standardizedPath = fileURL.standardizedFileURL.path + let previousSessionId = self.cachedFiles[path]?.sessionId + ?? self.cachedFiles[standardizedPath]?.sessionId + if let usage { + self.cachedFiles[path] = usage + self.cachedFiles[standardizedPath] = usage + } else { + self.cachedFiles.removeValue(forKey: path) + self.cachedFiles.removeValue(forKey: standardizedPath) + } + for sessionId in Set([previousSessionId, usage?.sessionId].compactMap(\.self)) { + self.snapshotResolutions.removeValue(forKey: sessionId) + self.resolvedDependencyKeys.removeValue(forKey: sessionId) + } } func inheritedTotals(for sessionId: String, atOrBefore cutoffTimestamp: String) throws -> CodexForkBaseline { @@ -814,30 +1381,112 @@ enum CostUsageScanner { "Codex cost usage could not parse fork timestamp; falling back to lexical comparison", metadata: ["sessionId": sessionId, "timestamp": cutoffTimestamp]) } - guard let snapshots = try self.snapshotResolution(for: sessionId).snapshots else { return .unresolved } - var inherited: CostUsageCodexTotals? - for snapshot in snapshots { - let isAtOrBefore: Bool = if let snapshotDate = snapshot.date, let cutoffDate { - snapshotDate <= cutoffDate + let resolution = try self.snapshotResolution(for: sessionId) + guard resolution.hasSnapshotSource else { return .unresolved } + if !resolution.isComplete { + guard let lastTimestamp = resolution.lastTimestamp else { return .unresolved } + let lastDate = CostUsageScanner.dateFromTimestamp(lastTimestamp) + let coversCutoff: Bool = if let lastDate, let cutoffDate { + lastDate >= cutoffDate } else { - snapshot.timestamp <= cutoffTimestamp - } - if isAtOrBefore { - inherited = snapshot.totals + lastTimestamp >= cutoffTimestamp } + guard coversCutoff else { return .unresolved } + } + let inherited = self.inheritedTotals( + from: resolution, + cutoffTimestamp: cutoffTimestamp, + cutoffDate: cutoffDate) + if let dependencyKey = resolution.dependencyKey { + self.resolvedDependencyKeys[sessionId] = dependencyKey } return .resolved(inherited) } - func currentDependencyKey(for sessionId: String) throws -> String { - guard let fileURL = try self.fileIndex.fileURL(for: sessionId) else { - return "missing:\(sessionId)" + private func inheritedTotals( + from resolution: SnapshotResolution, + cutoffTimestamp: String, + cutoffDate: Date?) -> CostUsageCodexTotals? + { + func isAtOrBefore(_ timestamp: String, date: Date? = nil) -> Bool { + if let date = date ?? CostUsageScanner.dateFromTimestamp(timestamp), let cutoffDate { + return date <= cutoffDate + } + return timestamp <= cutoffTimestamp + } + + if let events = resolution.indexedEvents { + var selectedCheckpoint: CostUsageCodexTokenCheckpoint? + let checkpointsAreSearchable = resolution.checkpoints.enumerated().allSatisfy { index, checkpoint in + checkpoint.eventIndex >= 0 + && checkpoint.eventIndex < events.count + && checkpoint.timestamp == events[checkpoint.eventIndex].timestamp + && (index == 0 + || resolution.checkpoints[index - 1].eventIndex < checkpoint.eventIndex) + } + let checkpoints = checkpointsAreSearchable ? resolution.checkpoints : [] + if resolution.indexedTimestampsMonotonic { + var lowerBound = 0 + var upperBound = checkpoints.count + while lowerBound < upperBound { + let middle = lowerBound + (upperBound - lowerBound) / 2 + if isAtOrBefore(checkpoints[middle].timestamp) { + lowerBound = middle + 1 + } else { + upperBound = middle + } + } + if lowerBound > 0 { + selectedCheckpoint = checkpoints[lowerBound - 1] + } + } else { + for checkpoint in checkpoints where isAtOrBefore(checkpoint.timestamp) { + selectedCheckpoint = checkpoint + } + } + + var accumulator = CodexSnapshotAccumulator(state: selectedCheckpoint?.state) + var inherited = selectedCheckpoint?.state.countedTotals + let startIndex = min(events.count, (selectedCheckpoint?.eventIndex ?? -1) + 1) + for event in events[startIndex...] { + let eventIsAtOrBefore = isAtOrBefore(event.timestamp) + if resolution.indexedTimestampsMonotonic, !eventIsAtOrBefore { + break + } + let counted = accumulator.apply(last: event.last, total: event.total) + if eventIsAtOrBefore { + inherited = counted + } + } + return inherited + } + + var inherited: CostUsageCodexTotals? + for snapshot in resolution.snapshots ?? [] where isAtOrBefore(snapshot.timestamp, date: snapshot.date) { + inherited = snapshot.totals + } + return inherited + } + + func currentDependencyKey(for sessionId: String) throws -> String? { + switch try self.fileIndex.lookup(sessionId: sessionId) { + case let .found(fileURL): + self.dependencyKey(for: sessionId, fileURL: fileURL) + case let .missing(dependencyKey): + dependencyKey + case .deferred: + nil } - return self.dependencyKey(for: sessionId, fileURL: fileURL) } func dependencyKeyUsed(for sessionId: String) -> String? { - self.snapshotResolutions[sessionId]?.dependencyKey + self.resolvedDependencyKeys[sessionId] + } + + func takePendingParentFiles() -> [URL] { + let files = self.pendingParentFiles.values.sorted(by: { $0.path < $1.path }) + self.pendingParentFiles.removeAll(keepingCapacity: true) + return files } private func dependencyKey(for sessionId: String, fileURL: URL) -> String { @@ -857,54 +1506,55 @@ enum CostUsageScanner { return cached } try self.checkCancellation?() - guard let fileURL = try self.fileIndex.fileURL(for: sessionId) else { + let lookup = try self.fileIndex.lookup(sessionId: sessionId) + let fileURL: URL + switch lookup { + case let .found(foundURL): + fileURL = foundURL + case let .missing(dependencyKey): CostUsageScanner.log.warning( "Codex cost usage parent session file not found", metadata: ["sessionId": sessionId]) let resolution = SnapshotResolution( - dependencyKey: "missing:\(sessionId)", - snapshots: nil) + dependencyKey: dependencyKey, + snapshots: nil, + isComplete: false) + self.snapshotResolutions[sessionId] = resolution + self.resolvedDependencyKeys[sessionId] = dependencyKey + return resolution + case .deferred: + let resolution = SnapshotResolution( + dependencyKey: nil, + snapshots: nil, + isComplete: false) self.snapshotResolutions[sessionId] = resolution return resolution } let parentMetadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) - if let budget = self.scanBudget { - switch budget.admit(workBytes: parentMetadata.size) { - case let .allow(allowance) where allowance >= parentMetadata.size: - break - case .allow: - CostUsageScanner.log.warning( - "Deferring oversized Codex parent baseline read while its file scan resumes", - metadata: [ - "sessionId": sessionId, - "path": fileURL.path, - "bytes": "\(parentMetadata.size)", - "slice": "\(budget.maxFileBytes)", - ]) - let resolution = SnapshotResolution( - dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), - snapshots: nil) - self.snapshotResolutions[sessionId] = resolution - return resolution - case .deferBudget: - CostUsageScanner.log.debug( - "Deferring Codex parent session baseline read until a later refresh", - metadata: [ - "sessionId": sessionId, - "path": fileURL.path, - "pendingBytes": "\(parentMetadata.size)", - "consumed": "\(budget.bytesConsumed)", - "limit": "\(budget.maxBytesPerRefresh)", - ]) - let resolution = SnapshotResolution( - dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), - snapshots: nil) - self.snapshotResolutions[sessionId] = resolution - return resolution - } + if let cachedResolution = self.cachedSnapshotResolution( + for: sessionId, + fileURL: fileURL, + metadata: parentMetadata) + { + self.snapshotResolutions[sessionId] = cachedResolution + return cachedResolution + } + if self.scanBudget != nil { + // A parent discovered while parsing a child must use the same persistent, + // resumable scan path as ordinary files. Queue it for this refresh instead of + // opening it here and bypassing the byte or wall-clock budget. + self.pendingParentFiles[fileURL.standardizedFileURL.path] = fileURL + let resolution = SnapshotResolution( + dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), + snapshots: nil, + isComplete: false) + self.snapshotResolutions[sessionId] = resolution + return resolution } + // Direct resolver construction without a scan budget is retained for focused parser + // tests and explicit unbounded callers. Production refreshes always install a budget. for _ in 0..<2 { let dependencyKeyBeforeParse = self.dependencyKey(for: sessionId, fileURL: fileURL) let parsed = try CostUsageScanner.parseCodexTokenSnapshots( @@ -919,7 +1569,8 @@ enum CostUsageScanner { metadata: ["sessionId": sessionId, "path": fileURL.path]) let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, - snapshots: nil) + snapshots: nil, + isComplete: false) self.snapshotResolutions[sessionId] = resolution self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution @@ -934,14 +1585,16 @@ enum CostUsageScanner { ]) let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, - snapshots: nil) + snapshots: nil, + isComplete: false) self.snapshotResolutions[sessionId] = resolution self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, - snapshots: parsed.snapshots) + snapshots: parsed.snapshots, + isComplete: true) self.snapshotResolutions[sessionId] = resolution self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution @@ -950,10 +1603,46 @@ enum CostUsageScanner { CostUsageScanner.log.warning( "Codex cost usage parent session changed while reading; deferring inherited baseline", metadata: ["sessionId": sessionId, "path": fileURL.path]) - let resolution = SnapshotResolution(dependencyKey: nil, snapshots: nil) + let resolution = SnapshotResolution(dependencyKey: nil, snapshots: nil, isComplete: false) self.snapshotResolutions[sessionId] = resolution return resolution } + + private func cachedSnapshotResolution( + for sessionId: String, + fileURL: URL, + metadata: CodexFileMetadata) -> SnapshotResolution? + { + let standardizedPath = fileURL.standardizedFileURL.path + let cachedUsage = self.cachedFiles[fileURL.path] ?? self.cachedFiles[standardizedPath] + guard let usage = cachedUsage, + usage.sessionId == sessionId, + usage.codexScanFileId == nil || usage.codexScanFileId == metadata.fileId, + let cachedSnapshots = usage.codexTokenSnapshots + else { return nil } + + let metadataMatches = usage.mtimeUnixMs == metadata.mtimeUnixMs + && usage.size == metadata.size + let appendSafePrefixMatches = usage.codexScanFileId == metadata.fileId + && usage.size <= metadata.size + && usage.codexTokenIndexAnchor.map { + CostUsageScanner.codexTokenIndexAnchorMatches( + $0, + fileURL: fileURL, + metadata: metadata) + } == true + guard metadataMatches || appendSafePrefixMatches else { return nil } + + let indexedBytes = usage.codexTokenIndexAnchor?.indexedBytes ?? usage.parsedBytes ?? usage.size + let coversCurrentFile = usage.codexScanComplete != false + && indexedBytes >= metadata.size + return SnapshotResolution( + dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), + indexedEvents: cachedSnapshots, + checkpoints: usage.codexTokenCheckpoints ?? [], + indexedTimestampsMonotonic: usage.codexTokenTimestampsMonotonic == true, + isComplete: coversCurrentFile) + } } struct ClaudeParseResult { @@ -1348,6 +2037,47 @@ enum CostUsageScanner { SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() } + static func codexTokenIndexAnchor( + fileURL: URL, + indexedBytes: Int64) -> CostUsageCodexTokenIndexAnchor? + { + let indexedBytes = max(0, indexedBytes) + guard indexedBytes > 0 else { return nil } + let windowStart = max(0, indexedBytes - 64 * 1024) + let byteCount = Int(indexedBytes - windowStart) + guard byteCount > 0 else { return nil } + + do { + let handle = try FileHandle(forReadingFrom: fileURL) + defer { try? handle.close() } + try handle.seek(toOffset: UInt64(windowStart)) + guard let data = try handle.read(upToCount: byteCount), data.count == byteCount else { + return nil + } + return CostUsageCodexTokenIndexAnchor( + indexedBytes: indexedBytes, + windowStart: windowStart, + sha256: Self.sha256Hex(data)) + } catch { + return nil + } + } + + static func codexTokenIndexAnchorMatches( + _ anchor: CostUsageCodexTokenIndexAnchor, + fileURL: URL, + metadata: CodexFileMetadata) -> Bool + { + guard anchor.indexedBytes > 0, + anchor.windowStart >= 0, + anchor.windowStart < anchor.indexedBytes, + metadata.size >= anchor.indexedBytes + else { return false } + return self.codexTokenIndexAnchor( + fileURL: fileURL, + indexedBytes: anchor.indexedBytes) == anchor + } + private static func listCodexRecentlyModifiedFiles( root: URL, scanSinceKey: String, @@ -1563,6 +2293,7 @@ enum CostUsageScanner { let forkTimestamp: String? let projectPath: String? let isSubagentThread: Bool + let subagentHistoryStartOrdinal: Int? } struct CodexTurnContextMetadata: Codable { @@ -1599,7 +2330,16 @@ enum CostUsageScanner { struct CodexBufferedFastLine: Codable { let lineIndex: Int + let ordinal: Int? + let endOffset: Int64? let line: CodexFastLine + + init(lineIndex: Int, ordinal: Int?, endOffset: Int64? = nil, line: CodexFastLine) { + self.lineIndex = lineIndex + self.ordinal = ordinal + self.endOffset = endOffset + self.line = line + } } private static let codexJSONFieldCachedInputTokens = Array("cached_input_tokens".utf8) @@ -1613,12 +2353,15 @@ enum CostUsageScanner { private static let codexJSONFieldModel = Array("model".utf8) private static let codexJSONFieldModelName = Array("model_name".utf8) private static let codexJSONFieldOutputTokens = Array("output_tokens".utf8) + private static let codexJSONFieldOrdinal = Array("ordinal".utf8) private static let codexJSONFieldReasoningOutputTokens = Array("reasoning_output_tokens".utf8) private static let codexJSONFieldParentSessionId = Array("parent_session_id".utf8) private static let codexJSONFieldParentSessionIdCamel = Array("parentSessionId".utf8) private static let codexJSONFieldPayload = Array("payload".utf8) private static let codexJSONFieldSource = Array("source".utf8) private static let codexJSONFieldSubagent = Array("subagent".utf8) + private static let codexJSONFieldSubagentHistoryStartOrdinal = + Array("subagent_history_start_ordinal".utf8) private static let codexJSONFieldSessionId = Array("session_id".utf8) private static let codexJSONFieldSessionIdCamel = Array("sessionId".utf8) private static let codexJSONFieldTimestamp = Array("timestamp".utf8) @@ -1874,7 +2617,14 @@ enum CostUsageScanner { projectPath: Self.codexProjectPath(from: rawBuffer, payloadRange: payloadRange), isSubagentThread: payloadRange.map { Self.codexIsSubagentThread(from: rawBuffer, in: $0) - } ?? false)) + } ?? false, + subagentHistoryStartOrdinal: payloadRange.flatMap { + Self.extractJSONByteIntField( + Self.codexJSONFieldSubagentHistoryStartOrdinal, + from: rawBuffer, + in: $0, + atDepth: 1) + })) case "turn_context": let timestamp = Self.extractJSONByteStringField( @@ -2051,6 +2801,18 @@ enum CostUsageScanner { return (Self.dayKeyFromTimestamp(timestamp) ?? Self.dayKeyFromParsedISO(timestamp)) != nil } + private static func codexLineOrdinal(_ bytes: Data) -> Int? { + bytes.withUnsafeBytes { rawBytes in + let rawBuffer = rawBytes.bindMemory(to: UInt8.self) + guard !rawBuffer.isEmpty else { return nil } + return Self.extractJSONByteIntField( + Self.codexJSONFieldOrdinal, + from: rawBuffer, + in: 0.. String? @@ -2074,7 +2836,8 @@ enum CostUsageScanner { forkTimestamp: payload?["timestamp"] as? String ?? obj["timestamp"] as? String, projectPath: Self.normalizedCodexProjectPath(payload?["cwd"] as? String), - isSubagentThread: Self.codexIsSubagentThread(from: payload)) + isSubagentThread: Self.codexIsSubagentThread(from: payload), + subagentHistoryStartOrdinal: (payload?["subagent_history_start_ordinal"] as? NSNumber)?.intValue) } private static func parseCodexSessionMetadata( @@ -2348,8 +3111,10 @@ enum CostUsageScanner { startedAtUnixMs: nil, latestActivityUnixMs: nil), rows: [], + tokenSnapshots: [], jsonlResumeState: nil, - bufferedSubagentLines: nil) + bufferedSubagentLines: nil, + bufferedUnresolvedForkLines: nil) } // swiftlint:disable:next cyclomatic_complexity function_body_length @@ -2367,8 +3132,10 @@ enum CostUsageScanner { initialCodexTurnID: String? = nil, initialCodexUsageRowIndex: Int = 0, initialBufferedSubagentLines: [CodexBufferedFastLine]? = nil, + initialBufferedUnresolvedForkLines: [CodexBufferedFastLine]? = nil, initialJSONLResumeState: CostUsageJsonl.ResumeState? = nil, maxBytesToRead: Int64? = nil, + shouldStopReading: ((Int64) -> Bool)? = nil, inheritedTotalsResolver: ((String, String) throws -> CodexForkBaseline)? = nil, checkCancellation: CancellationCheck? = nil) throws -> CodexParseResult { @@ -2380,6 +3147,7 @@ enum CostUsageScanner { var isSubagentThread = false var didCaptureLeafMetadata = false var forkTimestamp: String? + var subagentHistoryStartOrdinal: Int? var subagentCounterSemantics: CodexSubagentCounterSemantics? var usesLocalSubagentBoundary = false var candidateBoundaryDependsOnParentTotals = false @@ -2396,7 +3164,6 @@ enum CostUsageScanner { var remainingInheritedTotals: CostUsageCodexTotals? var forkBaselineResolved = false var hasUnresolvedForkBaseline = false - var unresolvedForkTotalWatermark: CostUsageCodexTotals? var currentTurnID = initialCodexTurnID var codexUsageRowIndex = initialCodexUsageRowIndex var rawTotalsBaseline = initialRawTotalsBaseline ?? initialTotals @@ -2409,6 +3176,7 @@ enum CostUsageScanner { var days: [String: [String: [Int]]] = [:] var rows: [CodexUsageRow] = [] + var tokenSnapshots: [CostUsageCodexTokenSnapshot] = [] func add(dayKey: String, model: String, input: Int, cached: Int, output: Int) { guard CostUsageDayRange.isInRange(dayKey: dayKey, since: range.scanSinceKey, until: range.scanUntilKey) @@ -2506,6 +3274,9 @@ enum CostUsageScanner { if projectPath == nil { projectPath = metadata.projectPath } + if subagentHistoryStartOrdinal == nil { + subagentHistoryStartOrdinal = metadata.subagentHistoryStartOrdinal + } observeTimestamp(metadata.forkTimestamp) if codexSession.cwd == nil { observeCwd(metadata.projectPath) @@ -2517,6 +3288,7 @@ enum CostUsageScanner { forkedFromId = metadata.forkedFromId forkTimestamp = metadata.forkTimestamp projectPath = metadata.projectPath + subagentHistoryStartOrdinal = metadata.subagentHistoryStartOrdinal codexSession.sessionId = metadata.sessionId codexSession.forkedFromId = metadata.forkedFromId observeTimestamp(metadata.forkTimestamp) @@ -2538,6 +3310,10 @@ enum CostUsageScanner { ?? CostUsagePricing.codexUnattributedModel let total = record.total let last = record.last + // A cumulative fork counter is not attributable until either the parent snapshot or + // a trustworthy child-owned suffix establishes the inherited baseline. Publishing + // best-effort `last` rows here can replay billions of copied-prefix tokens. + guard !hasUnresolvedForkBaseline else { return } var deltaInput = 0 var deltaCached = 0 @@ -2628,38 +3404,7 @@ enum CostUsageScanner { } } - let handledUnresolvedForkTotal = hasUnresolvedForkBaseline && total != nil - if hasUnresolvedForkBaseline, let total { - // `unresolvedForkTotalWatermark` is a presence sentinel for "skip the first - // unresolved-fork totals row"; delta baselines come from the global tracker. - let currentRawTotals = total - defer { - unresolvedForkTotalWatermark = currentRawTotals - } - guard let last, - unresolvedForkTotalWatermark != nil - else { - return - } - - let adjustedDelta = Self.codexMinTotals( - last, - Self.codexTotalDelta(from: watermarkBaseline, to: currentRawTotals)) - deltaInput = adjustedDelta.input - deltaCached = adjustedDelta.cached - deltaOutput = adjustedDelta.output - deltaReasoning = adjustedDelta.reasoning - let prev = previousTotals ?? .init( - input: 0, - cached: 0, - output: 0, - reasoning: adjustedDelta.reasoning == nil ? nil : 0) - previousTotals = Self.codexAddTotals(prev, adjustedDelta) - rawTotalsBaseline = previousTotals - } - - if !handledUnresolvedForkTotal, - let currentTotals = adjustedTotal, + if let currentTotals = adjustedTotal, forkedFromId != nil, !hasUnresolvedForkBaseline { @@ -2676,7 +3421,7 @@ enum CostUsageScanner { } commitDelta(delta, rawBaseline: currentTotals) remainingInheritedTotals = nil - } else if !handledUnresolvedForkTotal, let last { + } else if let last { let rawDelta = last let hadRemainingInheritedTotals = remainingInheritedTotals != nil var adjustedDelta = adjustedLastDelta(rawDelta) @@ -2719,10 +3464,10 @@ enum CostUsageScanner { rawTotalsBaseline = countedTotals tracker.raiseWatermark(to: countedTotals) } - } else if !handledUnresolvedForkTotal, let currentTotals = adjustedTotal { + } else if let currentTotals = adjustedTotal { commitDelta(totalsDerivedDelta(to: currentTotals), rawBaseline: currentTotals) remainingInheritedTotals = nil - } else if !handledUnresolvedForkTotal { + } else { return } @@ -2782,6 +3527,7 @@ enum CostUsageScanner { let prefixBytes = maxLineBytes var pendingSubagentLines = initialBufferedSubagentLines + var bufferedUnresolvedForkLines = initialBufferedUnresolvedForkLines if let initialBufferedSubagentLines, startOffset > 0 { for buffered in initialBufferedSubagentLines { @@ -2800,12 +3546,47 @@ enum CostUsageScanner { pendingSubagentLines = [] } } + if let initialBufferedUnresolvedForkLines, startOffset > 0 { + for buffered in initialBufferedUnresolvedForkLines { + guard case let .sessionMeta(metadata) = buffered.line else { continue } + try handleSessionMetadata(metadata) + } + if !hasUnresolvedForkBaseline { + for buffered in initialBufferedUnresolvedForkLines { + try processFastLine(buffered.line) + } + bufferedUnresolvedForkLines = nil + } + } - func routeFastLine(_ fastLine: CodexFastLine, lineIndex: Int) throws { + func routeFastLine( + _ fastLine: CodexFastLine, + lineIndex: Int, + ordinal: Int?, + endOffset: Int64) throws + { + let bufferedLine = Self.CodexBufferedFastLine( + lineIndex: lineIndex, + ordinal: ordinal, + endOffset: endOffset, + line: fastLine) + if case let .tokenCount(record) = fastLine, record.last != nil || record.total != nil { + tokenSnapshots.append(CostUsageCodexTokenSnapshot( + timestamp: record.timestamp, + last: record.last, + total: record.total, + endOffset: endOffset)) + } if pendingSubagentLines != nil { - pendingSubagentLines?.append(Self.CodexBufferedFastLine(lineIndex: lineIndex, line: fastLine)) + pendingSubagentLines?.append(bufferedLine) } else { try processFastLine(fastLine) + if hasUnresolvedForkBaseline { + if bufferedUnresolvedForkLines == nil { + bufferedUnresolvedForkLines = [] + } + bufferedUnresolvedForkLines?.append(bufferedLine) + } } } @@ -2821,6 +3602,7 @@ enum CostUsageScanner { prefixBytes: prefixBytes, maxBytesToRead: maxBytesToRead, resumeState: initialJSONLResumeState, + shouldStop: shouldStopReading, checkCancellation: checkCancellation, onLine: { line in let lineIndex = physicalLineIndex @@ -2842,7 +3624,9 @@ enum CostUsageScanner { model: truncatedTurnContext.model, cwd: nil, title: nil)), - lineIndex: lineIndex) + lineIndex: lineIndex, + ordinal: nil, + endOffset: line.endOffset) } catch { deferredError = error } @@ -2857,8 +3641,11 @@ enum CostUsageScanner { forkedFromId: nil, forkTimestamp: nil, projectPath: nil, - isSubagentThread: false)), - lineIndex: lineIndex) + isSubagentThread: false, + subagentHistoryStartOrdinal: nil)), + lineIndex: lineIndex, + ordinal: nil, + endOffset: line.endOffset) } catch { deferredError = error } @@ -2885,12 +3672,17 @@ enum CostUsageScanner { } if let fastLine = Self.parseCodexFastLine(line.bytes) { + let ordinal = Self.codexLineOrdinal(line.bytes) let timestampValidity = fastLine.requiresValidTimestamp ? Self.codexFastLineTimestampValidity(line.bytes) : true if timestampValidity == true { do { - try routeFastLine(fastLine, lineIndex: lineIndex) + try routeFastLine( + fastLine, + lineIndex: lineIndex, + ordinal: ordinal, + endOffset: line.endOffset) } catch { deferredError = error } @@ -2906,11 +3698,16 @@ enum CostUsageScanner { let obj = (try? JSONSerialization.jsonObject(with: line.bytes)) as? [String: Any], let type = obj["type"] as? String else { return } + let ordinal = (obj["ordinal"] as? NSNumber)?.intValue if type == "session_meta" { guard let metadata = Self.codexSessionMetadata(from: obj) else { return } do { - try routeFastLine(.sessionMeta(metadata), lineIndex: lineIndex) + try routeFastLine( + .sessionMeta(metadata), + lineIndex: lineIndex, + ordinal: ordinal, + endOffset: line.endOffset) } catch { deferredError = error } @@ -2926,7 +3723,9 @@ enum CostUsageScanner { do { try routeFastLine( .interAgentCommunication(triggerTurn: payload?["trigger_turn"] as? Bool == true), - lineIndex: lineIndex) + lineIndex: lineIndex, + ordinal: ordinal, + endOffset: line.endOffset) } catch { deferredError = error } @@ -2954,7 +3753,11 @@ enum CostUsageScanner { title: payload["title"] as? String ?? payload["name"] as? String) } do { - try routeFastLine(.turnContext(metadata), lineIndex: lineIndex) + try routeFastLine( + .turnContext(metadata), + lineIndex: lineIndex, + ordinal: ordinal, + endOffset: line.endOffset) } catch { deferredError = error } @@ -2967,7 +3770,9 @@ enum CostUsageScanner { do { try routeFastLine( .taskStarted(turnID: Self.codexTurnID(from: payload)), - lineIndex: lineIndex) + lineIndex: lineIndex, + ordinal: ordinal, + endOffset: line.endOffset) } catch { deferredError = error } @@ -3005,7 +3810,11 @@ enum CostUsageScanner { last: (info?["last_token_usage"] as? [String: Any]).map(tokenTotals), total: (info?["total_token_usage"] as? [String: Any]).map(tokenTotals)) do { - try routeFastLine(.tokenCount(record), lineIndex: lineIndex) + try routeFastLine( + .tokenCount(record), + lineIndex: lineIndex, + ordinal: ordinal, + endOffset: line.endOffset) } catch { deferredError = error } @@ -3033,6 +3842,9 @@ enum CostUsageScanner { if projectPath == nil { projectPath = metadata.projectPath } + if subagentHistoryStartOrdinal == nil { + subagentHistoryStartOrdinal = metadata.subagentHistoryStartOrdinal + } observeTimestamp(metadata.forkTimestamp) if codexSession.cwd == nil { observeCwd(metadata.projectPath) @@ -3065,21 +3877,66 @@ enum CostUsageScanner { if forkedFromId == nil { forkedFromId = shape.inferredParentSessionID } - var ownedSuffix = shape.ownedSuffix - if let candidate = shape.ownedSuffixCandidate, - let parentSessionID = forkedFromId - { - candidateBoundaryDependsOnParentTotals = true - if let inheritedTotalsResolver { - switch try inheritedTotalsResolver(parentSessionID, forkTimestamp ?? "") { - case let .resolved(parentTotals): - if Self.codexTotalsEqual(parentTotals, candidate.parentTotalsAtBoundary) { - subagentCounterSemantics = .copiedPrefix - ownedSuffix = candidate.ownedSuffix - parentConfirmedLocalBoundary = true + let explicitOwnedSuffix: CodexSubagentRolloutShape.CodexSubagentOwnedSuffix? = { + guard let startOrdinal = subagentHistoryStartOrdinal, + let firstOwnedLine = pendingSubagentLines.first(where: { + ($0.ordinal ?? Int.min) >= startOrdinal + }) + else { return nil } + + let inheritedTotal = pendingSubagentLines + .prefix(while: { ($0.ordinal ?? Int.min) < startOrdinal }) + .compactMap { buffered -> CostUsageCodexTotals? in + guard case let .tokenCount(record) = buffered.line else { return nil } + return record.total + } + .last + let firstOwnedToken = pendingSubagentLines.first { buffered in + guard (buffered.ordinal ?? Int.min) >= startOrdinal, + case .tokenCount = buffered.line + else { return false } + return true + } + let inferredTotal = firstOwnedToken.flatMap { buffered -> CostUsageCodexTotals? in + guard case let .tokenCount(record) = buffered.line else { return nil } + if let total = record.total, let last = record.last, + Self.codexTotalsAtLeast(total, last) + { + return Self.codexTotalDelta(from: last, to: total) + } + if record.total == nil, record.last != nil { + return .init(input: 0, cached: 0, output: 0) + } + return nil + } + guard let rawTotalsBaseline = inheritedTotal ?? inferredTotal else { return nil } + return .init( + startLineIndex: firstOwnedLine.lineIndex, + rawTotalsBaseline: rawTotalsBaseline) + }() + + var ownedSuffix = explicitOwnedSuffix ?? shape.ownedSuffix + var locallyConfirmedBoundary = explicitOwnedSuffix != nil + if explicitOwnedSuffix != nil { + subagentCounterSemantics = .copiedPrefix + } else if let candidate = shape.ownedSuffixCandidate { + if candidate.isLocallyConfirmed { + subagentCounterSemantics = .copiedPrefix + ownedSuffix = candidate.ownedSuffix + locallyConfirmedBoundary = true + } else if let parentSessionID = forkedFromId { + candidateBoundaryDependsOnParentTotals = true + if let inheritedTotalsResolver { + switch try inheritedTotalsResolver(parentSessionID, forkTimestamp ?? "") { + case let .resolved(parentTotals): + if Self.codexTotalsEqual(parentTotals, candidate.parentTotalsAtBoundary) { + subagentCounterSemantics = .copiedPrefix + ownedSuffix = candidate.ownedSuffix + parentConfirmedLocalBoundary = true + } + case .unresolved: + break } - case .unresolved: - break } } } @@ -3100,7 +3957,6 @@ enum CostUsageScanner { sawInterleavedTotals: false) currentModel = nil currentTurnID = nil - unresolvedForkTotalWatermark = nil } self.log.debug( "Codex cost usage classified subagent rollout counter semantics", @@ -3108,6 +3964,7 @@ enum CostUsageScanner { "sessionId": sessionId ?? "unknown", "semantics": subagentCounterSemantics == .copiedPrefix ? "copiedPrefix" : "independent", "localBoundary": ownedSuffix == nil ? "false" : "true", + "locallyConfirmedBoundary": locallyConfirmedBoundary ? "true" : "false", "parentConfirmedBoundary": parentConfirmedLocalBoundary ? "true" : "false", "suppressedUnownedPrefix": suppressUnownedCopiedPrefix ? "true" : "false", "sessionMetadataCount": String(observations.count(where: { @@ -3157,8 +4014,16 @@ enum CostUsageScanner { projectPath: projectPath, codexSession: codexSession, rows: rows, + tokenSnapshots: tokenSnapshots, jsonlResumeState: jsonlResumeState, - bufferedSubagentLines: parsedBytes < targetSize || jsonlResumeState != nil ? pendingSubagentLines : nil) + bufferedSubagentLines: parsedBytes < targetSize + || jsonlResumeState != nil + || hasUnresolvedForkBaseline + ? pendingSubagentLines + : nil, + bufferedUnresolvedForkLines: hasUnresolvedForkBaseline + ? bufferedUnresolvedForkLines + : nil) } private static func codexTurnID(from payload: [String: Any]) -> String? { @@ -3228,6 +4093,7 @@ enum CostUsageScanner { if fullRescanWorkBytes == pendingWorkBytes { fullRescanAllowedBytes = allowedWorkBytes } else if let budget = context.scanBudget { + budget.release(workBytes: allowedWorkBytes) switch budget.admit(workBytes: fullRescanWorkBytes) { case let .allow(allowance): fullRescanAllowedBytes = allowance @@ -3254,13 +4120,29 @@ enum CostUsageScanner { // (forced full rescan, priority invalidation, fork-dependency drift, etc.), the scanner // will read the whole file — never report zero pending work in that case. guard let cached else { return max(0, metadata.size) } + if cached.forkedFromId != nil, + cached.forkBaselineDependencyKey == nil, + cached.hasBufferedCodexForkRetryLines, + cached.codexScanFileId == metadata.fileId, + cached.parsedBytes == metadata.size + { + return 0 + } if cached.codexScanComplete == false { if cached.codexScanFileId != nil, cached.codexScanFileId == metadata.fileId, - cached.codexScanTargetSize == metadata.size, - cached.mtimeUnixMs == metadata.mtimeUnixMs + let parsedBytes = cached.parsedBytes, + parsedBytes > 0, + parsedBytes <= metadata.size, + cached.codexTokenIndexAnchor?.indexedBytes == parsedBytes, + cached.codexTokenIndexAnchor.map({ + Self.codexTokenIndexAnchorMatches( + $0, + fileURL: URL(fileURLWithPath: metadata.path), + metadata: metadata) + }) == true { - return max(0, metadata.size - (cached.parsedBytes ?? 0)) + return max(0, metadata.size - parsedBytes) } return max(0, metadata.size) } @@ -3370,13 +4252,76 @@ enum CostUsageScanner { shouldRefresh: shouldRefresh) } - private static func loadCodexCache(options: Options, range: CostUsageDayRange) -> CostUsageCache { - CostUsageCacheIO.load( - provider: .codex, + private static func loadCodexCache( + options: Options, + range: CostUsageDayRange) -> CostUsageCodexCacheLoadResult + { + CostUsageCacheIO.loadCodexForMigration( cacheRoot: options.cacheRoot, calendar: range.calendar) } + private static func codexPreviousReportCandidate( + cache: CostUsageCache, + incompatibleCache: CostUsageCache?, + range: CostUsageDayRange, + plan: CodexRefreshPlan, + options: Options) -> CostUsageCodexPreviousReport? + { + let currentScanIsPending = cache.codexScanCatchUpPending == true + || cache.files.values.contains { $0.codexScanComplete == false } + || cache.files.values.contains { $0.hasBufferedCodexForkRetryLines } + if currentScanIsPending, + let previous = self.codexPreviousReport( + cache: cache, + range: range, + rootsFingerprint: plan.rootsFingerprint) + { + return previous + } + + let sourceCache: CostUsageCache? = if let incompatibleCache { + incompatibleCache + } else if !currentScanIsPending, + options.forceRescan, + !cache.days.isEmpty + { + cache + } else { + nil + } + guard let sourceCache, + sourceCache.timeZoneIdentifier == range.calendar.timeZone.identifier, + sourceCache.roots == plan.rootsFingerprint, + !self.requestedWindowExpandsCache(range: range, cache: sourceCache), + !sourceCache.days.isEmpty + else { return nil } + + let report = self.buildCodexReportFromCache( + cache: sourceCache, + range: range, + modelsDevCatalog: plan.modelsDevCatalog, + modelsDevCacheRoot: options.cacheRoot, + priorityTurns: plan.priorityTurns) + return CostUsageCodexPreviousReport(report: report, cache: sourceCache) + } + + static func codexPreviousReport( + cache: CostUsageCache, + range: CostUsageDayRange, + rootsFingerprint: [String: Int64]) -> CostUsageCodexPreviousReport? + { + guard cache.codexScanCatchUpPending == true, + let previous = cache.codexPreviousReport, + previous.matches( + scanSinceKey: range.scanSinceKey, + scanUntilKey: range.scanUntilKey, + timeZoneIdentifier: range.calendar.timeZone.identifier, + roots: rootsFingerprint) + else { return nil } + return previous + } + private static func saveCodexCache(_ cache: CostUsageCache, options: Options, range: CostUsageDayRange) { CostUsageCacheIO.save( provider: .codex, @@ -3392,9 +4337,16 @@ enum CostUsageScanner { options: Options, checkCancellation: CancellationCheck?) throws -> CostUsageDailyReport { - var cache = Self.loadCodexCache(options: options, range: range) + let loadedCache = Self.loadCodexCache(options: options, range: range) + var cache = loadedCache.cache let nowMs = Int64(now.timeIntervalSince1970 * 1000) let plan = Self.makeCodexRefreshPlan(cache: cache, range: range, now: now, nowMs: nowMs, options: options) + let previousReport = Self.codexPreviousReportCandidate( + cache: cache, + incompatibleCache: loadedCache.incompatibleCache, + range: range, + plan: plan, + options: options) if plan.shouldRefresh { try checkCancellation?() @@ -3451,8 +4403,11 @@ enum CostUsageScanner { files = Self.sortedCodexSessionFilesNewestFirst(files) } - let filePathsInScan = Set(files.map(\.path)) - var scanState = CodexScanState() + var filePathsInScan = Set(files.map(\.path)) + let scanBudget = CodexScanBudget( + maxFileBytes: options.maxCodexSessionFileBytes, + maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh, + maxDuration: options.maxCodexScanDurationPerRefresh) let fileIndex = CodexSessionFileIndex( files: files, roots: plan.roots, @@ -3460,14 +4415,15 @@ enum CostUsageScanner { cache: cache, roots: plan.roots, knownExistingPaths: filePathsInScan), + cachedDiscovery: plan.rootsChanged ? nil : cache.codexSessionDiscovery, + scanBudget: scanBudget, + headParseObserver: self.codexSessionHeadParseObserverStore?.observer, checkCancellation: checkCancellation) - let scanBudget = CodexScanBudget( - maxFileBytes: options.maxCodexSessionFileBytes, - maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh) let inheritedResolver = CodexInheritedTotalsResolver( fileIndex: fileIndex, checkCancellation: checkCancellation, - scanBudget: scanBudget) + scanBudget: scanBudget, + cachedFiles: cache.files) let resources = CodexScanResources( fileIndex: fileIndex, inheritedResolver: inheritedResolver, @@ -3482,19 +4438,21 @@ enum CostUsageScanner { resources: resources, checkCancellation: checkCancellation, scanBudget: scanBudget) - for fileURL in files { - try Self.scanCodexFile( - fileURL: fileURL, - context: scanContext, - cache: &cache, - state: &scanState) - } - if scanBudget.resumedPartialFileCount > 0 || scanBudget.deferredByBudgetFileCount > 0 { + try filePathsInScan.formUnion(Self.scanCodexFiles( + files, + context: scanContext, + cache: &cache, + inheritedResolver: inheritedResolver)) + if scanBudget.resumedPartialFileCount > 0 + || scanBudget.deferredByBudgetFileCount > 0 + || scanBudget.deferredByTimeBudgetFileCount > 0 + { Self.log.info( "Codex cost scan applied work limits", metadata: [ "partialFiles": "\(scanBudget.resumedPartialFileCount)", "deferredByBudget": "\(scanBudget.deferredByBudgetFileCount)", + "deferredByTime": "\(scanBudget.deferredByTimeBudgetFileCount)", "bytesConsumed": "\(scanBudget.bytesConsumed)", "maxFileBytes": "\(scanBudget.maxFileBytes)", "maxBytesPerRefresh": "\(scanBudget.maxBytesPerRefresh)", @@ -3546,6 +4504,21 @@ enum CostUsageScanner { cache.codexPricingKey = plan.codexPricingKey cache.codexPriorityMetadataKey = plan.codexPriorityMetadataKey cache.codexProjectMetadataVersion = Self.codexProjectMetadataVersion + let scanProgress = Self.codexScanProgress(paths: filePathsInScan, cache: cache) + cache.codexScanProcessedBytes = scanProgress.processedBytes + cache.codexScanTotalBytes = scanProgress.totalBytes + cache.codexScanCompletedFiles = scanProgress.completedFiles + cache.codexScanTotalFiles = scanProgress.totalFiles + cache.codexSessionDiscovery = fileIndex.persistedState + let catchUpPending = scanBudget.resumedPartialFileCount > 0 + || scanBudget.deferredByBudgetFileCount > 0 + || scanBudget.deferredByTimeBudgetFileCount > 0 + || scanProgress.completedFiles < scanProgress.totalFiles + || cache.files.values.contains { $0.codexScanComplete == false } + || cache.files.values.contains { $0.hasBufferedCodexForkRetryLines } + || fileIndex.hasPendingDiscovery + cache.codexScanCatchUpPending = catchUpPending + cache.codexPreviousReport = catchUpPending ? previousReport : nil if plan.hasPriorityMetadata { cache.codexPriorityTurnKeys = Self.mergePriorityTurnKeys( existing: shouldRetainWiderWindow ? cache.codexPriorityTurnKeys : nil, @@ -3565,6 +4538,13 @@ enum CostUsageScanner { Self.saveCodexCache(cache, options: options, range: range) } + if let previous = Self.codexPreviousReport( + cache: cache, + range: range, + rootsFingerprint: plan.rootsFingerprint) + { + return previous.report + } return Self.buildCodexReportFromCache( cache: cache, range: range, @@ -3573,6 +4553,127 @@ enum CostUsageScanner { priorityTurns: plan.priorityTurns) } + private struct CodexScanProgressSummary { + let processedBytes: Int64 + let totalBytes: Int64 + let completedFiles: Int + let totalFiles: Int + } + + private static func codexScanProgress( + paths: Set, + cache: CostUsageCache) -> CodexScanProgressSummary + { + var processedBytes: Int64 = 0 + var totalBytes: Int64 = 0 + var completedFiles = 0 + var totalFiles = 0 + var seenIdentities: Set = [] + + for path in paths.sorted() { + let fileURL = URL(fileURLWithPath: path) + let metadata = Self.codexFileMetadata(fileURL: fileURL) + let identity = metadata.fileId ?? fileURL.standardizedFileURL.path + guard seenIdentities.insert(identity).inserted else { continue } + totalFiles += 1 + totalBytes += max(0, metadata.size) + + let usage = cache.files[path] ?? cache.files[fileURL.standardizedFileURL.path] + guard let usage else { continue } + let identityMatches = usage.codexScanFileId == nil || usage.codexScanFileId == metadata.fileId + guard identityMatches else { continue } + let parsedBytes = min( + max(0, metadata.size), + max(0, usage.parsedBytes ?? (usage.codexScanComplete == false ? 0 : usage.size))) + processedBytes += parsedBytes + if usage.codexScanComplete != false, + parsedBytes >= metadata.size, + !usage.hasBufferedCodexForkRetryLines + { + completedFiles += 1 + } + } + + return CodexScanProgressSummary( + processedBytes: processedBytes, + totalBytes: totalBytes, + completedFiles: completedFiles, + totalFiles: totalFiles) + } + + private static func scanCodexFiles( + _ files: [URL], + context: CodexFileScanContext, + cache: inout CostUsageCache, + inheritedResolver: CodexInheritedTotalsResolver) throws -> Set + { + var scanState = CodexScanState() + var bufferedForkRetries: [URL] = [] + var visitedPaths = Set(files.map(\.standardizedFileURL.path)) + var scannedPaths = Set(files.map(\.path)) + for fileURL in files { + try Self.scanCodexFile( + fileURL: fileURL, + context: context, + cache: &cache, + state: &scanState) + let usage = cache.files[fileURL.path] + inheritedResolver.updateCachedUsage(fileURL: fileURL, usage: usage) + if Self.shouldRetryBufferedCodexFork(usage) { + bufferedForkRetries.append(fileURL) + } + } + + // Parents outside the requested history window are discovered only after parsing their + // children. Scan those dependencies through the same budgeted path and retain their cache + // entries so later passes can resume instead of restarting from byte zero. + var dependencyState = CodexScanState() + while true { + let pendingParents = inheritedResolver.takePendingParentFiles().filter { + visitedPaths.insert($0.standardizedFileURL.path).inserted + } + guard !pendingParents.isEmpty else { break } + for fileURL in pendingParents { + scannedPaths.insert(fileURL.path) + try Self.scanCodexFile( + fileURL: fileURL, + context: context, + cache: &cache, + state: &dependencyState) + let usage = cache.files[fileURL.path] + inheritedResolver.updateCachedUsage(fileURL: fileURL, usage: usage) + if Self.shouldRetryBufferedCodexFork(usage) { + bufferedForkRetries.append(fileURL) + } + } + } + + // Newest-first ordering commonly encounters a child before its parent. Once this + // refresh has indexed the parent, replay the child's compact parsed events in memory; + // do not reread the JSONL or wait for another refresh. + var retryState = CodexScanState() + var retriedPaths: Set = [] + for fileURL in bufferedForkRetries where retriedPaths.insert(fileURL.path).inserted { + guard Self.shouldRetryBufferedCodexFork(cache.files[fileURL.path]) else { continue } + try Self.scanCodexFile( + fileURL: fileURL, + context: context, + cache: &cache, + state: &retryState) + inheritedResolver.updateCachedUsage( + fileURL: fileURL, + usage: cache.files[fileURL.path]) + } + return scannedPaths + } + + private static func shouldRetryBufferedCodexFork(_ usage: CostUsageFileUsage?) -> Bool { + guard let usage else { return false } + return usage.forkedFromId != nil + && usage.forkBaselineDependencyKey == nil + && usage.hasBufferedCodexForkRetryLines + } + private static func codexFileScanContext( range: CostUsageDayRange, options: Options, @@ -3595,9 +4696,12 @@ enum CostUsageScanner { } static func sortedCodexSessionFilesNewestFirst(_ files: [URL]) -> [URL] { - files.sorted { lhs, rhs in - let left = Self.codexFileMetadata(fileURL: lhs) - let right = Self.codexFileMetadata(fileURL: rhs) + let metadata = files.reduce(into: [String: CodexFileMetadata]()) { result, fileURL in + result[fileURL.path] = Self.codexFileMetadata(fileURL: fileURL) + } + return files.sorted { lhs, rhs in + let left = metadata[lhs.path] ?? Self.codexFileMetadata(fileURL: lhs) + let right = metadata[rhs.path] ?? Self.codexFileMetadata(fileURL: rhs) if left.mtimeUnixMs != right.mtimeUnixMs { return left.mtimeUnixMs > right.mtimeUnixMs } diff --git a/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift b/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift index 3fa37a5bb9..29f9ac44ae 100644 --- a/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift +++ b/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift @@ -7,7 +7,7 @@ struct CodexCompactSubagentAccountingTests { private typealias Usage = Fixture.Usage @Test - func `parent-confirmed first turn marker drops a compact copied prefix`() throws { + func `locally confirmed first turn marker drops a compact copied prefix`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -85,11 +85,11 @@ struct CodexCompactSubagentAccountingTests { CostUsagePricing.normalizeCodexModel(leafModel), ] == [50, 10, 5]) #expect(child.days.values.allSatisfy { $0[CostUsagePricing.codexUnattributedModel] == nil }) - #expect(child.forkBaselineDependencyKey?.hasPrefix("file|") == true) + #expect(child.forkBaselineDependencyKey == CostUsageScanner.codexForkDependencyNotRequiredKey) } @Test - func `parent snapshot change invalidates a cached compact child classification`() throws { + func `parent snapshot change keeps a locally confirmed compact child cached`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -135,14 +135,14 @@ struct CodexCompactSubagentAccountingTests { now: day, options: options) let beforeDay = try #require(before.data.first) - #expect(beforeDay.totalTokens == 2253) - #expect(beforeDay.modelBreakdowns?.first { + #expect(beforeDay.totalTokens == 1153) + #expect(!(beforeDay.modelBreakdowns ?? []).contains { $0.modelName == CostUsagePricing.codexUnattributedModel - }?.totalTokens == 1100) + }) let beforeCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) let beforeChild = try #require(beforeCache.files.values.first { $0.sessionId == "cache-child" }) let beforeDependency = try #require(beforeChild.forkBaselineDependencyKey) - #expect(beforeDependency.hasPrefix("file|")) + #expect(beforeDependency == CostUsageScanner.codexForkDependencyNotRequiredKey) let appendedParentSnapshot = try env.jsonl([ Fixture.tokenCount( @@ -167,13 +167,12 @@ struct CodexCompactSubagentAccountingTests { }) let afterCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) let afterChild = try #require(afterCache.files.values.first { $0.sessionId == "cache-child" }) - #expect(afterChild.forkBaselineDependencyKey?.hasPrefix("file|") == true) - #expect(afterChild.forkBaselineDependencyKey != beforeDependency) + #expect(afterChild.forkBaselineDependencyKey == beforeDependency) #expect(afterChild.days.values.allSatisfy { $0[CostUsagePricing.codexUnattributedModel] == nil }) } @Test - func `unconfirmed compact prefix stays independent and parent-dependent`() throws { + func `locally confirmed compact prefix ignores parent resolution`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -205,9 +204,9 @@ struct CodexCompactSubagentAccountingTests { range: CostUsageScanner.CostUsageDayRange(since: day, until: day), inheritedTotalsResolver: { _, _ in baseline }) let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) - #expect(parsed.days[dayKey]?[CostUsagePricing.codexUnattributedModel] == [1000, 900, 100]) + #expect(parsed.days[dayKey]?[CostUsagePricing.codexUnattributedModel] == nil) #expect(parsed.days[dayKey]?[CostUsagePricing.normalizeCodexModel(leafModel)] == [50, 10, 5]) - #expect(parsed.dependsOnParentTotals) + #expect(!parsed.dependsOnParentTotals) } } } diff --git a/Tests/CodexBarTests/CodexCostCatchUpPolicyTests.swift b/Tests/CodexBarTests/CodexCostCatchUpPolicyTests.swift new file mode 100644 index 0000000000..a58c367070 --- /dev/null +++ b/Tests/CodexBarTests/CodexCostCatchUpPolicyTests.swift @@ -0,0 +1,69 @@ +import Foundation +import Testing +@testable import CodexBar + +struct CodexCostCatchUpPolicyTests { + @Test + func `automatic mode targets twenty percent duty cycle on AC power`() { + let decision = CodexCostCatchUpPolicy().decision(for: .init( + mode: .automatic, + previousActiveDuration: 2, + powerSource: .ac, + lowPowerModeEnabled: false, + thermalState: .nominal)) + + #expect(decision == .init(action: .runAfter(8), targetDutyCycle: 0.2)) + } + + @Test + func `automatic mode targets five percent duty cycle on battery`() { + let decision = CodexCostCatchUpPolicy().decision(for: .init( + mode: .automatic, + previousActiveDuration: 2, + powerSource: .battery, + lowPowerModeEnabled: false, + thermalState: .nominal)) + + guard case let .runAfter(delay) = decision.action else { + Issue.record("Expected automatic battery catch-up to schedule another pass") + return + } + #expect(abs(delay - 38) < 0.000_001) + #expect(decision.targetDutyCycle == 0.05) + } + + @Test + func `automatic mode pauses for low power mode`() { + let decision = CodexCostCatchUpPolicy().decision(for: .init( + mode: .automatic, + previousActiveDuration: 2, + powerSource: .battery, + lowPowerModeEnabled: true, + thermalState: .nominal)) + + #expect(decision == .init( + action: .pause(CodexCostCatchUpPolicy.constrainedRetryDelay, .lowPower), + targetDutyCycle: nil)) + } + + @Test + func `accelerated mode ignores low power but not critical thermal pressure`() { + let lowPowerDecision = CodexCostCatchUpPolicy().decision(for: .init( + mode: .accelerated, + previousActiveDuration: 2, + powerSource: .battery, + lowPowerModeEnabled: true, + thermalState: .serious)) + let criticalDecision = CodexCostCatchUpPolicy().decision(for: .init( + mode: .accelerated, + previousActiveDuration: 2, + powerSource: .ac, + lowPowerModeEnabled: false, + thermalState: .critical)) + + #expect(lowPowerDecision == .init(action: .runAfter(0), targetDutyCycle: 1)) + #expect(criticalDecision == .init( + action: .pause(CodexCostCatchUpPolicy.constrainedRetryDelay, .thermal), + targetDutyCycle: nil)) + } +} diff --git a/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift b/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift index 91eeebb957..932fda7740 100644 --- a/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift +++ b/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift @@ -506,6 +506,137 @@ struct CodexSubagentAccountingIntegrationTests { #expect(parsed.rows.isEmpty) } + @Test + func `protocol ordinal isolates the child suffix without resolving its parent`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 7, day: 16) + let timestamp = env.isoString(for: day) + let parentModel = "openai/gpt-5.3" + let leafModel = "openai/gpt-5.4" + var sessionMetadata: [String: Any] = [ + "type": "session_meta", + "timestamp": timestamp, + "payload": [ + "id": "ordinal-child", + "forked_from_id": "ordinal-parent", + "timestamp": timestamp, + "subagent_history_start_ordinal": 10, + "source": [ + "subagent": [ + "thread_spawn": ["parent_thread_id": "ordinal-parent"], + ], + ], + ], + ] + sessionMetadata["ordinal"] = 0 + var copiedTotal = self.tokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(1)), + model: parentModel, + total: (input: 1000, cached: 900, output: 100), + last: (input: 50, cached: 10, output: 5)) + copiedTotal["ordinal"] = 9 + var ownedContext = self.turnContext( + timestamp: env.isoString(for: day.addingTimeInterval(2)), + model: leafModel) + ownedContext["ordinal"] = 10 + var ownedTotal = self.tokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(3)), + model: leafModel, + total: (input: 1050, cached: 910, output: 105), + last: (input: 50, cached: 10, output: 5)) + ownedTotal["ordinal"] = 11 + let fileURL = try env.writeCodexSessionFile( + day: day, + filename: "rollout-\(timestamp)-ordinal-child.jsonl", + contents: env.jsonl([sessionMetadata, copiedTotal, ownedContext, ownedTotal])) + + var resolvedParentBaseline = false + let parsed = CostUsageScanner.parseCodexFile( + fileURL: fileURL, + range: CostUsageScanner.CostUsageDayRange(since: day, until: day), + inheritedTotalsResolver: { _, _ in + resolvedParentBaseline = true + return .resolved(.init(input: 1, cached: 1, output: 1)) + }) + + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + let normalizedLeafModel = CostUsagePricing.normalizeCodexModel(leafModel) + #expect(parsed.days[dayKey]?[normalizedLeafModel] == [50, 10, 5]) + #expect(parsed.days[dayKey]?[CostUsagePricing.normalizeCodexModel(parentModel)] == nil) + #expect(!parsed.dependsOnParentTotals) + #expect(!resolvedParentBaseline) + } + + @Test + func `legacy child proves its inherited baseline from first owned total minus last`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 7, day: 16) + let timestamp = env.isoString(for: day) + let parentModel = "openai/gpt-5.3" + let leafModel = "openai/gpt-5.4" + let fileURL = try env.writeCodexSessionFile( + day: day, + filename: "rollout-\(timestamp)-legacy-self-confirmed.jsonl", + contents: env.jsonl([ + [ + "type": "session_meta", + "timestamp": timestamp, + "payload": [ + "id": "legacy-self-confirmed", + "forked_from_id": "legacy-parent", + "timestamp": timestamp, + "source": [ + "subagent": [ + "thread_spawn": ["parent_thread_id": "legacy-parent"], + ], + ], + ], + ], + self.tokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(1)), + model: parentModel, + total: (input: 1000, cached: 900, output: 100)), + self.turnContext( + timestamp: env.isoString(for: day.addingTimeInterval(2)), + model: leafModel), + [ + "type": "inter_agent_communication_metadata", + "timestamp": env.isoString(for: day.addingTimeInterval(2)), + "payload": ["trigger_turn": true], + ], + self.tokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(3)), + model: leafModel, + total: (input: 1050, cached: 910, output: 105), + last: (input: 50, cached: 10, output: 5)), + self.tokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(4)), + model: leafModel, + total: (input: 1070, cached: 915, output: 110), + last: (input: 20, cached: 5, output: 5)), + ])) + + var resolvedParentBaseline = false + let parsed = CostUsageScanner.parseCodexFile( + fileURL: fileURL, + range: CostUsageScanner.CostUsageDayRange(since: day, until: day), + inheritedTotalsResolver: { _, _ in + resolvedParentBaseline = true + return .unresolved + }) + + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + let normalizedLeafModel = CostUsagePricing.normalizeCodexModel(leafModel) + #expect(parsed.days[dayKey]?[normalizedLeafModel] == [70, 15, 10]) + #expect(parsed.days[dayKey]?[CostUsagePricing.normalizeCodexModel(parentModel)] == nil) + #expect(!parsed.dependsOnParentTotals) + #expect(!resolvedParentBaseline) + } + @Test func `bounded append fallback reclassifies the complete subagent rollout`() throws { let env = try CostUsageTestEnvironment() diff --git a/Tests/CodexBarTests/CostUsageCacheTests.swift b/Tests/CodexBarTests/CostUsageCacheTests.swift index d06d66c09c..ddd3a1af94 100644 --- a/Tests/CodexBarTests/CostUsageCacheTests.swift +++ b/Tests/CodexBarTests/CostUsageCacheTests.swift @@ -139,6 +139,13 @@ struct CostUsageCacheTests { #expect(stale.lastScanUnixMs == 0) #expect(stale.files.isEmpty) #expect(stale.days.isEmpty) + + let migration = CostUsageCacheIO.loadCodexForMigration( + cacheRoot: root, + producerKey: "codex:cu:p2222222222222222") + #expect(migration.cache.days.isEmpty) + #expect(migration.incompatibleCache?.producerKey == "codex:cu:p1111111111111111") + #expect(migration.incompatibleCache?.days["2026-05-18"]?["gpt-5.5"] == [1, 2, 3]) } @Test diff --git a/Tests/CodexBarTests/CostUsageFetcherCacheSnapshotTests.swift b/Tests/CodexBarTests/CostUsageFetcherCacheSnapshotTests.swift index bcf9d25777..685fea2a03 100644 --- a/Tests/CodexBarTests/CostUsageFetcherCacheSnapshotTests.swift +++ b/Tests/CodexBarTests/CostUsageFetcherCacheSnapshotTests.swift @@ -35,6 +35,47 @@ struct CostUsageFetcherCacheSnapshotTests { #expect(cached?.daily.map(\.date) == ["2026-04-08"]) } + @Test + func `cached codex token snapshot exposes an incompatible producer as stale upgrade data`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 4, day: 8) + try Self.writeCodexSessionFile( + homeRoot: env.codexHomeRoot, + env: env, + day: day, + filename: "cached.jsonl", + tokens: 42) + + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + _ = try await CostUsageFetcher.loadTokenSnapshot( + provider: .codex, + now: day, + historyDays: 1, + scannerOptions: options) + + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let scanTime = Date(timeIntervalSince1970: TimeInterval(cache.lastScanUnixMs) / 1000) + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: env.cacheRoot, + producerKey: "codex:cu:pupgrade-fixture") + + let cached = await CostUsageFetcher.loadCachedCodexTokenSnapshotResult( + now: day.addingTimeInterval(60), + historyDays: 1, + scannerOptions: options) + + #expect(cached?.snapshot.sessionTokens == 42) + #expect(cached?.snapshot.updatedAt == scanTime) + #expect(cached?.lastRefreshAt == nil) + #expect(cached?.staleSnapshotUpdatedAt == scanTime) + } + @Test func `cached codex token snapshot keeps the cache scan time as updatedAt`() async throws { let env = try CostUsageTestEnvironment() diff --git a/Tests/CodexBarTests/CostUsageFetcherTests.swift b/Tests/CodexBarTests/CostUsageFetcherTests.swift index 5db8108693..909807640d 100644 --- a/Tests/CodexBarTests/CostUsageFetcherTests.swift +++ b/Tests/CodexBarTests/CostUsageFetcherTests.swift @@ -857,6 +857,26 @@ extension CostUsageFetcherTests { #expect(refreshed.daily.first?.totalTokens == 176) } + @Test + func `app codex refresh bounds its initial scan before background catch up`() { + #expect(CostUsageFetcher.resolvedCodexScanDurationPerRefresh( + provider: .codex, + bypassScannerDebounce: true, + configuredDuration: nil) == 2) + #expect(CostUsageFetcher.resolvedCodexScanDurationPerRefresh( + provider: .codex, + bypassScannerDebounce: false, + configuredDuration: nil) == nil) + #expect(CostUsageFetcher.resolvedCodexScanDurationPerRefresh( + provider: .claude, + bypassScannerDebounce: true, + configuredDuration: nil) == nil) + #expect(CostUsageFetcher.resolvedCodexScanDurationPerRefresh( + provider: .codex, + bypassScannerDebounce: true, + configuredDuration: 7) == 7) + } + private static func writeCodexSessionFile( homeRoot: URL, env: CostUsageTestEnvironment, diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 926b623b02..9c6a646504 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -13,21 +13,25 @@ struct CostUsageJsonlScannerTests { let contents = "\(largeLine)\nsmall\n" try contents.write(to: fileURL, atomically: true, encoding: .utf8) - var scanned: [(count: Int, truncated: Bool)] = [] + var scanned: [(count: Int, truncated: Bool, start: Int64, end: Int64)] = [] let endOffset = try CostUsageJsonl.scan( fileURL: fileURL, maxLineBytes: 400_000, prefixBytes: 400_000) { line in - scanned.append((line.bytes.count, line.wasTruncated)) + scanned.append((line.bytes.count, line.wasTruncated, line.startOffset, line.endOffset)) } #expect(endOffset == Int64(Data(contents.utf8).count)) #expect(scanned.count == 2) #expect(scanned[0].count == 300_000) #expect(scanned[0].truncated == false) + #expect(scanned[0].start == 0) + #expect(scanned[0].end == 300_001) #expect(scanned[1].count == 5) #expect(scanned[1].truncated == false) + #expect(scanned[1].start == 300_001) + #expect(scanned[1].end == 300_007) } @Test @@ -556,6 +560,49 @@ struct CostUsageJsonlScannerTests { #expect(endOffset == Int64(Data(record.utf8).count)) } + @Test + func `bounded jsonl scanner yields after a chunk and resumes the same record`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("time-sliced.jsonl", isDirectory: false) + let first = #"{"message":"\#(String(repeating: "x", count: 300_000))"}"# + let second = #"{"message":"done"}"# + try "\(first)\n\(second)\n".write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPassLines: [CostUsageJsonl.Line] = [] + let firstProgress = try CostUsageJsonl.scanBounded( + fileURL: fileURL, + maxLineBytes: 400_000, + prefixBytes: 400_000, + maxBytesToRead: nil, + resumeState: nil, + shouldStop: { $0 >= 256 * 1024 }, + onLine: { line in + firstPassLines.append(line) + }) + + #expect(firstPassLines.isEmpty) + #expect(firstProgress.readOffset == 256 * 1024) + #expect(firstProgress.committedOffset == 0) + let resumeState = try #require(firstProgress.resumeState) + + var resumedLines: [String] = [] + let completed = try CostUsageJsonl.scanBounded( + fileURL: fileURL, + maxLineBytes: 400_000, + prefixBytes: 400_000, + maxBytesToRead: nil, + resumeState: resumeState, + onLine: { line in + resumedLines.append(String(data: line.bytes, encoding: .utf8) ?? "") + }) + + #expect(resumedLines == [first, second]) + #expect(completed.resumeState == nil) + #expect(completed.committedOffset == Int64(Data("\(first)\n\(second)\n".utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", diff --git a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift index c202d20157..504004e9dd 100644 --- a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift +++ b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift @@ -390,7 +390,7 @@ struct CostUsagePerformanceGateTests { } @Test - func `oversized codex progress restarts when the target size changes`() throws { + func `oversized codex progress survives an append while catch-up is in progress`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) @@ -433,13 +433,202 @@ struct CostUsagePerformanceGateTests { until: day, now: day, options: options) - let restarted = try #require(CostUsageCacheIO.load( + let resumed = try #require(CostUsageCacheIO.load( provider: .codex, cacheRoot: env.cacheRoot).files.values.first) - #expect(restarted.parsedBytes == slice) - #expect(restarted.codexScanTargetSize == changedMetadata.size) - #expect(restarted.codexScanFileId == changedMetadata.fileId) - #expect(restarted.codexScanComplete == false) + #expect((resumed.parsedBytes ?? 0) > (first.parsedBytes ?? 0)) + #expect(resumed.parsedBytes == min(changedMetadata.size, (first.parsedBytes ?? 0) + slice)) + #expect(resumed.codexScanTargetSize == changedMetadata.size) + #expect(resumed.codexScanFileId == changedMetadata.fileId) + #expect(resumed.codexScanComplete == false) + } + + @Test + func `catch-up API continuously advances bounded slices to the exact full result`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let files = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 1, turnsPerFile: 8) + let fileURL = try #require(files.first) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + let slice = max(1, metadata.size / 4) + + var baselineOptions = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.root.appendingPathComponent("baseline-cache"), + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0) + baselineOptions.refreshMinIntervalSeconds = 0 + let baseline = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: baselineOptions) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: slice, + maxCodexScanBytesPerRefresh: slice) + options.refreshMinIntervalSeconds = 0 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + let fetcher = CostUsageFetcher(scannerOptions: options) + var status = await fetcher.codexScanCatchUpStatus() + #expect(status.pending) + var progressKeys = [status.progressKey] + for _ in 0..<12 where status.pending { + status = try await fetcher.advanceCodexScanCatchUp(now: day, historyDays: 1) + progressKeys.append(status.progressKey) + } + + let completedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let completedUsage = try #require(completedCache.files.values.first) + let completedReport = CostUsageScanner.buildCodexReportFromCache( + cache: completedCache, + range: CostUsageScanner.CostUsageDayRange(since: day, until: day)) + #expect(!status.pending) + #expect(completedUsage.codexScanComplete == true) + #expect(completedUsage.parsedBytes == metadata.size) + #expect(zip(progressKeys, progressKeys.dropFirst()).allSatisfy(!=)) + #expect(completedReport.summary?.totalTokens == baseline.summary?.totalTokens) + #expect(completedReport.data.map(\.totalTokens) == baseline.data.map(\.totalTokens)) + } + + @Test + func `incompatible populated cache stays visible until bounded fork rebuild converges`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + let forkISO = env.isoString(for: day.addingTimeInterval(2)) + let model = "openai/gpt-5.2-codex" + + let parentBody = ([ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"upgrade-parent"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"\#(model)"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":500,"cached_input_tokens":50,"output_tokens":25},"# + + #""model":"\#(model)"}}}"#, + ] + Array(repeating: "x", count: 4096)).joined(separator: "\n") + "\n" + let parentURL = try env.writeCodexSessionFile( + day: day, + filename: "upgrade-parent.jsonl", + contents: parentBody) + let childBody = [ + #"{"type":"session_meta","timestamp":"\#(forkISO)","payload":{"session_id":"upgrade-child","# + + #""forked_from_id":"upgrade-parent"}}"#, + #"{"type":"turn_context","timestamp":"\#(forkISO)","payload":{"model":"\#(model)"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":600,"cached_input_tokens":60,"output_tokens":30},"# + + #""model":"\#(model)"}}}"#, + ].joined(separator: "\n") + "\n" + let childURL = try env.writeCodexSessionFile( + day: day, + filename: "upgrade-child.jsonl", + contents: childBody) + try FileManager.default.setAttributes( + [.modificationDate: day], + ofItemAtPath: parentURL.path) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(120)], + ofItemAtPath: childURL.path) + + var baselineOptions = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.root.appendingPathComponent("upgrade-baseline-cache"), + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0) + baselineOptions.refreshMinIntervalSeconds = 0 + let baseline = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: baselineOptions) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 1024, + maxCodexScanBytesPerRefresh: 1024, + preferNewestCodexSessionsFirst: true) + options.refreshMinIntervalSeconds = 0 + let range = CostUsageScanner.CostUsageDayRange( + since: day, + until: day, + calendar: options.calendar) + let priorScanAt = day.addingTimeInterval(-3600) + var priorCache = CostUsageCache() + priorCache.lastScanUnixMs = Int64(priorScanAt.timeIntervalSince1970 * 1000) + priorCache.scanSinceKey = range.scanSinceKey + priorCache.scanUntilKey = range.scanUntilKey + priorCache.timeZoneIdentifier = options.calendar.timeZone.identifier + priorCache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + priorCache.days = [ + range.sinceKey: [CostUsagePricing.normalizeCodexModel(model): [777, 0, 0]], + ] + CostUsageCacheIO.save( + provider: .codex, + cache: priorCache, + cacheRoot: env.cacheRoot, + producerKey: "codex:cu:pupgrade-fixture") + + let priorReport = CostUsageScanner.buildCodexReportFromCache( + cache: priorCache, + range: range) + var report = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let fetcher = CostUsageFetcher(scannerOptions: options) + var status = await fetcher.codexScanCatchUpStatus() + + #expect(status.pending) + #expect(status.staleSnapshotUpdatedAt == priorScanAt) + #expect(report.data == priorReport.data) + #expect(report.summary == priorReport.summary) + #expect(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).codexPreviousReport != nil) + + for pass in 1...16 where status.pending { + report = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(TimeInterval(pass)), + options: options) + status = await fetcher.codexScanCatchUpStatus() + if status.pending { + #expect(report.data == priorReport.data) + #expect(report.summary == priorReport.summary) + #expect(status.staleSnapshotUpdatedAt == priorScanAt) + } + } + + let completedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + #expect(!status.pending) + #expect(status.staleSnapshotUpdatedAt == nil) + #expect(completedCache.codexPreviousReport == nil) + #expect(report.data == baseline.data) + #expect(report.summary == baseline.summary) } @Test @@ -532,6 +721,298 @@ struct CostUsagePerformanceGateTests { #expect(budget.bytesConsumed == 150) } + @Test + func `codex scan budget yields after its wall clock deadline`() { + let clock = TestMonotonicClock() + let budget = CostUsageScanner.CodexScanBudget( + maxFileBytes: 100, + maxBytesPerRefresh: 150, + maxDuration: 2, + now: { clock.now() }) + guard case let .allow(first) = budget.admit(workBytes: 100) else { + Issue.record("expected work before the deadline to be admitted") + return + } + budget.consume(workBytes: first) + + clock.advance(by: .seconds(3)) + #expect(budget.shouldYield(additionalBytes: 0)) + #expect(budget.deferredByTimeBudgetFileCount == 1) + #expect(budget.shouldYield(additionalBytes: 0)) + #expect(budget.deferredByTimeBudgetFileCount == 1) + } +} + +private final class TestMonotonicClock: @unchecked Sendable { + private let lock = NSLock() + private let origin = ContinuousClock.now + private var offset = Duration.zero + + func now() -> ContinuousClock.Instant { + self.lock.withLock { + self.origin.advanced(by: self.offset) + } + } + + func advance(by duration: Duration) { + self.lock.withLock { + self.offset += duration + } + } +} + +extension CostUsagePerformanceGateTests { + @Test + func `missing parent head discovery resumes inside the scan budget`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + let body = #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"known-session","cwd":"# + + String(repeating: "x", count: 512) + + #""}}"# + + "\n" + let fileURL = try env.writeCodexSessionFile(day: day, filename: "budgeted-head.jsonl", contents: body) + + var discovery: CostUsageCodexSessionDiscovery? + var offsets: [Int64] = [] + var resolvedMissing = false + for _ in 0..<32 { + let budget = CostUsageScanner.CodexScanBudget(maxFileBytes: 32, maxBytesPerRefresh: 32) + let index = CostUsageScanner.CodexSessionFileIndex( + files: [fileURL], + roots: [env.codexSessionsRoot], + cachedDiscovery: discovery, + scanBudget: budget) + switch try index.lookup(sessionId: "absent-session") { + case .found: + Issue.record("unexpected parent resolution") + case .missing: + resolvedMissing = true + case .deferred: + break + } + discovery = index.persistedState + if let offset = discovery?.headScan?.resumeState?.offset ?? discovery?.headScan?.offset { + offsets.append(offset) + } + if resolvedMissing { + break + } + } + + #expect(offsets.count >= 2) + #expect(offsets[1] > offsets[0]) + #expect(resolvedMissing) + #expect(discovery?.missingSessionIds.contains("absent-session") == true) + } + + @Test + func `missing fork parent stays idle then publishes buffered usage once when created`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + let forkISO = env.isoString(for: day.addingTimeInterval(1)) + _ = try Self.writeSyntheticCodexCorpus( + env: env, + day: day, + files: 250, + turnsPerFile: 0) + + let childBody = [ + #"{"type":"session_meta","timestamp":"\#(forkISO)","payload":{"session_id":"missing-child","# + + #""forked_from_id":"late-parent"}}"#, + #"{"type":"turn_context","timestamp":"\#(forkISO)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":150,"cached_input_tokens":15,"output_tokens":8},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, + ].joined(separator: "\n") + "\n" + let childURL = try env.writeCodexSessionFile( + day: day, + filename: "missing-child.jsonl", + contents: childBody) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(600)], + ofItemAtPath: childURL.path) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite")) + options.refreshMinIntervalSeconds = 0 + + let coldCounter = HeadParseCounter() + _ = CostUsageScanner.withCodexSessionHeadParseObserverForTesting { + coldCounter.increment() + } operation: { + CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + } + let coldCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let coldChild = try #require(coldCache.files.values.first { $0.sessionId == "missing-child" }) + let coldDiscovery = try #require(coldCache.codexSessionDiscovery) + #expect(coldCounter.value >= 250) + #expect(coldChild.days.isEmpty) + #expect(coldChild.forkBaselineDependencyKey?.contains("missing|late-parent|discovery|") == true) + #expect(coldDiscovery.missingSessionIds.contains("late-parent")) + + let warmCounter = HeadParseCounter() + _ = CostUsageScanner.withCodexSessionHeadParseObserverForTesting { + warmCounter.increment() + } operation: { + CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + } + #expect(warmCounter.value == 0) + + let parentBody = [ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"late-parent"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":100,"cached_input_tokens":10,"output_tokens":5},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, + ].joined(separator: "\n") + "\n" + _ = try env.writeCodexSessionFile(day: day, filename: "late-parent.jsonl", contents: parentBody) + + let resolved = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let resolvedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let resolvedChild = try #require(resolvedCache.files.values.first { $0.sessionId == "missing-child" }) + #expect(!resolvedChild.days.isEmpty) + #expect(resolvedChild.forkBaselineDependencyKey?.hasPrefix("file|late-parent|") == true) + + let stable = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(3), + options: options) + let stableCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let stableChild = try #require(stableCache.files.values.first { $0.sessionId == "missing-child" }) + #expect(stableChild.days == resolvedChild.days) + #expect(stable.summary?.totalTokens == resolved.summary?.totalTokens) + } + + @Test + func `partition inventory change rotates negative lookup generation`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let forkISO = env.isoString(for: day.addingTimeInterval(1)) + _ = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 20, turnsPerFile: 0) + let childBody = [ + #"{"type":"session_meta","timestamp":"\#(forkISO)","payload":{"session_id":"inventory-child","# + + #""forked_from_id":"inventory-missing"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":10,"cached_input_tokens":1,"output_tokens":1}}}"#, + ].joined(separator: "\n") + "\n" + let childURL = try env.writeCodexSessionFile( + day: day, + filename: "inventory-child.jsonl", + contents: childBody) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(600)], + ofItemAtPath: childURL.path) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite")) + options.refreshMinIntervalSeconds = 0 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let firstCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let firstGeneration = try #require(firstCache.codexSessionDiscovery?.generation) + #expect(firstCache.codexSessionDiscovery?.missingSessionIds.contains("inventory-missing") == true) + + let newFile = try env.writeCodexSessionFile( + day: day, + filename: "inventory-new.jsonl", + contents: #"{"type":"session_meta","timestamp":"\#(forkISO)","payload":{"session_id":"new-session"}}"# + + "\n") + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(300)], + ofItemAtPath: newFile.path) + let counter = HeadParseCounter() + _ = CostUsageScanner.withCodexSessionHeadParseObserverForTesting { + counter.increment() + } operation: { + CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + } + let changedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let changedGeneration = try #require(changedCache.codexSessionDiscovery?.generation) + #expect(changedGeneration != firstGeneration) + #expect(changedCache.codexSessionDiscovery?.missingSessionIds.contains("inventory-missing") == true) + #expect(counter.value <= 1) + } + + @Test + func `sparse token checkpoints preserve the exact accumulator state`() { + let mebibyte: Int64 = 1024 * 1024 + let events = [ + CostUsageCodexTokenSnapshot( + timestamp: "2026-05-10T12:00:00Z", + last: .init(input: 100, cached: 10, output: 5), + total: .init(input: 100, cached: 10, output: 5), + endOffset: 1 * mebibyte), + CostUsageCodexTokenSnapshot( + timestamp: "2026-05-10T12:01:00Z", + last: .init(input: 100, cached: 10, output: 5), + total: .init(input: 200, cached: 20, output: 10), + endOffset: 5 * mebibyte), + CostUsageCodexTokenSnapshot( + timestamp: "2026-05-10T12:02:00Z", + last: .init(input: 10, cached: 1, output: 1), + total: .init(input: 150, cached: 15, output: 8), + endOffset: 6 * mebibyte), + CostUsageCodexTokenSnapshot( + timestamp: "2026-05-10T12:03:00Z", + last: .init(input: 70, cached: 7, output: 2), + total: .init(input: 220, cached: 22, output: 12), + endOffset: 10 * mebibyte), + CostUsageCodexTokenSnapshot( + timestamp: "2026-05-10T12:04:00Z", + last: .init(input: 10, cached: 1, output: 1), + total: .init(input: 230, cached: 23, output: 13), + endOffset: 10 * mebibyte + 1), + ] + + let checkpoints = CostUsageScanner.codexTokenCheckpoints(for: events) + #expect(checkpoints.map(\.eventIndex) == [1, 3, 4]) + + for checkpoint in checkpoints { + var accumulator = CostUsageScanner.CodexSnapshotAccumulator() + for event in events[...checkpoint.eventIndex] { + _ = accumulator.apply(last: event.last, total: event.total) + } + #expect(checkpoint.state == accumulator.state) + } + } + @Test func `per refresh byte budget defers later dirty files`() throws { let env = try CostUsageTestEnvironment() @@ -609,27 +1090,32 @@ struct CostUsagePerformanceGateTests { } @Test - func `oversized parent baseline reads defer for small fork children`() throws { + func `oversized parent baseline resolves from its same refresh partial snapshot`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) let iso = env.isoString(for: day) + let parentTotalISO = env.isoString(for: day.addingTimeInterval(1)) + let forkISO = env.isoString(for: day.addingTimeInterval(2)) // Parent is intentionally larger than the per-file budget. let parentBody = ([ #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"parent-giant"}}"#, #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, - #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"# + #"{"type":"event_msg","timestamp":"\#(parentTotalISO)","payload":{"type":"token_count","info":"# + #"{"total_token_usage":{"input_tokens":500,"cached_input_tokens":50,"output_tokens":25},"# + #""model":"openai/gpt-5.2-codex"}}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"last_token_usage":{"input_tokens":20,"cached_input_tokens":5,"output_tokens":3},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, ] + Array(repeating: "x", count: 4096)).joined(separator: "\n") + "\n" _ = try env.writeCodexSessionFile(day: day, filename: "parent-giant.jsonl", contents: parentBody) let childBody = [ - #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"child-small","# + #"{"type":"session_meta","timestamp":"\#(forkISO)","payload":{"session_id":"child-small","# + #""forked_from_id":"parent-giant"}}"#, - #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, - #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"# + #"{"type":"turn_context","timestamp":"\#(forkISO)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + #"{"total_token_usage":{"input_tokens":600,"cached_input_tokens":60,"output_tokens":30},"# + #""model":"openai/gpt-5.2-codex"}}}"#, ].joined(separator: "\n") + "\n" @@ -645,19 +1131,215 @@ struct CostUsagePerformanceGateTests { options.refreshMinIntervalSeconds = 0 let started = Date() - let report = CostUsageScanner.loadDailyReport( + _ = CostUsageScanner.loadDailyReport( provider: .codex, since: day, until: day, now: day, options: options) let elapsed = Date().timeIntervalSince(started) - let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let firstCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let firstParent = try #require(firstCache.files.values.first { $0.sessionId == "parent-giant" }) + let firstChild = try #require(firstCache.files.values.first { $0.sessionId == "child-small" }) + let firstChildDay = try #require( + firstChild.days[CostUsageScanner.CostUsageDayRange.dayKey(from: day)]) + let firstChildTokens = try #require( + firstChildDay[CostUsagePricing.normalizeCodexModel("openai/gpt-5.2-codex")]) #expect(elapsed < 2.0) - #expect(cache.files.keys.contains { URL(fileURLWithPath: $0).lastPathComponent == childURL.lastPathComponent }) - // Child still contributes local tokens while the parent baseline is unresolved. - #expect((report.summary?.totalTokens ?? 0) > 0) + #expect(firstCache.files.keys.contains { + URL(fileURLWithPath: $0).lastPathComponent == childURL.lastPathComponent + }) + #expect(firstChildTokens == [80, 5, 2]) + #expect(firstChild.forkBaselineDependencyKey != nil) + #expect(firstChild.codexBufferedSubagentLines == nil) + #expect(firstParent.codexScanComplete == false) + #expect(firstParent.codexTokenSnapshots?.count == 2) + #expect(firstParent.codexTokenSnapshots?.last?.last == .init(input: 20, cached: 5, output: 3)) + #expect(firstParent.codexTokenCheckpoints?.isEmpty == false) + #expect(firstParent.codexTokenIndexAnchor != nil) + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let secondCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let parent = try #require(secondCache.files.values.first { $0.sessionId == "parent-giant" }) + let child = try #require(secondCache.files.values.first { $0.sessionId == "child-small" }) + let childDay = try #require(child.days[CostUsageScanner.CostUsageDayRange.dayKey(from: day)]) + let childTokens = try #require( + childDay[CostUsagePricing.normalizeCodexModel("openai/gpt-5.2-codex")]) + + #expect(parent.codexScanComplete == false) + #expect(parent.codexTokenSnapshots?.count == 2) + #expect(childTokens == [80, 5, 2]) + #expect(child.forkBaselineDependencyKey != nil) + } + + @Test + func `appended parent resolves from a validated cached prefix without rereading it`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + let forkISO = env.isoString(for: day.addingTimeInterval(1)) + let appendedISO = env.isoString(for: day.addingTimeInterval(10)) + + let parentBody = [ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"parent-append"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":500,"cached_input_tokens":50,"output_tokens":25},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, + ].joined(separator: "\n") + "\n" + let parentURL = try env.writeCodexSessionFile( + day: day, + filename: "parent-append.jsonl", + contents: parentBody) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0) + options.refreshMinIntervalSeconds = 0 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + let indexedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let indexedParentEntry = try #require( + indexedCache.files.first { $0.value.sessionId == "parent-append" }) + let parentCachePath = indexedParentEntry.key + let indexedParent = indexedParentEntry.value + let indexedSize = indexedParent.size + #expect(indexedParent.codexScanComplete == true) + #expect(indexedParent.codexTokenIndexAnchor?.indexedBytes == indexedSize) + #expect(indexedParent.codexTokenCheckpoints?.isEmpty == false) + + let appendedLine = #"{"type":"event_msg","timestamp":"\#(appendedISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":900,"cached_input_tokens":90,"output_tokens":45},"# + + #""model":"openai/gpt-5.2-codex"}}}"# + "\n" + let parentHandle = try FileHandle(forWritingTo: parentURL) + try parentHandle.seekToEnd() + try parentHandle.write(contentsOf: Data(appendedLine.utf8)) + try parentHandle.close() + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(60)], + ofItemAtPath: parentURL.path) + + let childBody = [ + #"{"type":"session_meta","timestamp":"\#(forkISO)","payload":{"session_id":"child-append","# + + #""forked_from_id":"parent-append"}}"#, + #"{"type":"turn_context","timestamp":"\#(forkISO)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":600,"cached_input_tokens":60,"output_tokens":30},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, + ].joined(separator: "\n") + "\n" + let childURL = try env.writeCodexSessionFile( + day: day, + filename: "child-append.jsonl", + contents: childBody) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(120)], + ofItemAtPath: childURL.path) + let childSize = CostUsageScanner.codexFileMetadata(fileURL: childURL).size + + options.maxCodexSessionFileBytes = 64 * 1024 * 1024 + options.maxCodexScanBytesPerRefresh = childSize + options.preferNewestCodexSessionsFirst = true + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(120), + options: options) + + let refreshedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let deferredParent = try #require(refreshedCache.files[parentCachePath]) + let child = try #require( + refreshedCache.files.values.first { $0.sessionId == "child-append" }) + let childDay = try #require(child.days[CostUsageScanner.CostUsageDayRange.dayKey(from: day)]) + let childTokens = try #require( + childDay[CostUsagePricing.normalizeCodexModel("openai/gpt-5.2-codex")]) + + #expect(childTokens == [100, 10, 5]) + #expect(child.forkBaselineDependencyKey != nil) + #expect(deferredParent.size == indexedSize) + #expect(CostUsageScanner.codexFileMetadata(fileURL: parentURL).size > deferredParent.size) + } + + @Test + func `rewritten parent prefix rejects its cached token index`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + let forkISO = env.isoString(for: day.addingTimeInterval(1)) + let originalBody = [ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"parent-rewrite"}}"#, + #"{"type":"event_msg","timestamp":"\#(forkISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":500,"cached_input_tokens":50,"output_tokens":25}}}}"#, + ].joined(separator: "\n") + "\n" + let parentURL = try env.writeCodexSessionFile( + day: day, + filename: "parent-rewrite.jsonl", + contents: originalBody) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0) + options.refreshMinIntervalSeconds = 0 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let usage = try #require(cache.files.values.first { $0.sessionId == "parent-rewrite" }) + let anchor = try #require(usage.codexTokenIndexAnchor) + + let rewrittenBody = originalBody.replacingOccurrences( + of: #""input_tokens":500"#, + with: #""input_tokens":900"#) + #expect(rewrittenBody.utf8.count == originalBody.utf8.count) + try rewrittenBody.write(to: parentURL, atomically: false, encoding: .utf8) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(60)], + ofItemAtPath: parentURL.path) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: parentURL) + #expect(!CostUsageScanner.codexTokenIndexAnchorMatches( + anchor, + fileURL: parentURL, + metadata: metadata)) + + let fileIndex = CostUsageScanner.CodexSessionFileIndex( + files: [parentURL], + roots: [env.codexSessionsRoot]) + let resolver = CostUsageScanner.CodexInheritedTotalsResolver( + fileIndex: fileIndex, + checkCancellation: nil, + scanBudget: CostUsageScanner.CodexScanBudget(maxFileBytes: 1, maxBytesPerRefresh: 1), + cachedFiles: cache.files) + guard case .unresolved = try resolver.inheritedTotals( + for: "parent-rewrite", + atOrBefore: forkISO) + else { + Issue.record("rewritten prefix must not reuse the cached fork baseline") + return + } } private static func writeSyntheticCodexCorpus( @@ -677,12 +1359,17 @@ struct CostUsagePerformanceGateTests { #"{"type":"session_meta","timestamp":"\#(baseISO)","payload":{"session_id":"perf-\#(fileIndex)"}}"#) lines.append( #"{"type":"turn_context","timestamp":"\#(baseISO)","payload":{"model":"\#(model)"}}"#) - for turn in 1...turnsPerFile { - let inputTokens = turn * inputTokensPerTurn - lines.append( - #"{"type":"event_msg","timestamp":"\#(baseISO)","payload":{"type":"token_count","info":"# - + #"{"total_token_usage":{"input_tokens":\#(inputTokens),"cached_input_tokens":\#(turn * 20),"# - + #""output_tokens":\#(turn * 10)},"model":"\#(model)"}}}"#) + if turnsPerFile > 0 { + for turn in 1...turnsPerFile { + let inputTokens = turn * inputTokensPerTurn + let cachedTokens = turn * 20 + let outputTokens = turn * 10 + lines.append( + #"{"type":"event_msg","timestamp":"\#(baseISO)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":\#(inputTokens),"# + + #""cached_input_tokens":\#(cachedTokens),"output_tokens":\#(outputTokens)},"# + + #""model":"\#(model)"}}}"#) + } } let fileURL = try env.writeCodexSessionFile( day: day, @@ -715,4 +1402,17 @@ struct CostUsagePerformanceGateTests { } } } + +private final class HeadParseCounter: @unchecked Sendable { + private let lock = NSLock() + private var count = 0 + + var value: Int { + self.lock.withLock { self.count } + } + + func increment() { + self.lock.withLock { self.count += 1 } + } +} #endif diff --git a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift index 547cf7fc0e..a8b6474035 100644 --- a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift +++ b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift @@ -4222,9 +4222,7 @@ struct CostUsageScannerBreakdownTests { until: childDay, now: childDay, options: options) - #expect(withoutParent.data.first?.inputTokens == 7) - #expect(withoutParent.data.first?.cacheReadTokens == 2) - #expect(withoutParent.data.first?.outputTokens == 2) + #expect(withoutParent.data.isEmpty) _ = try env.writeCodexSessionFile( day: parentDay, @@ -4503,10 +4501,7 @@ struct CostUsageScannerBreakdownTests { now: childDay, options: options) - #expect(report.data.count == 1) - #expect(report.data[0].inputTokens == 20) - #expect(report.data[0].outputTokens == 3) - #expect(report.data[0].totalTokens == 23) + #expect(report.data.isEmpty) } @Test @@ -5054,7 +5049,7 @@ struct CostUsageScannerBreakdownTests { } @Test - func `codex unresolved fork ignores duplicated total and last replay after prefix`() throws { + func `codex unresolved fork remains fail closed after later total and last rows`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -5105,13 +5100,8 @@ struct CostUsageScannerBreakdownTests { return .unresolved }) - let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) - let normalized = CostUsagePricing.normalizeCodexModel(model) - let packed = try #require(parsed.days[dayKey]?[normalized]) - #expect(packed[0] == 30) - #expect(packed[1] == 7) - #expect(packed[2] == 8) - #expect(parsed.rows.count == 2) + #expect(parsed.days.isEmpty) + #expect(parsed.rows.isEmpty) } @Test diff --git a/Tests/CodexBarTests/Issue2037ScannerIntegrationTests.swift b/Tests/CodexBarTests/Issue2037ScannerIntegrationTests.swift index 17339b8b28..55cf0870be 100644 --- a/Tests/CodexBarTests/Issue2037ScannerIntegrationTests.swift +++ b/Tests/CodexBarTests/Issue2037ScannerIntegrationTests.swift @@ -125,7 +125,7 @@ struct Issue2037ScannerIntegrationTests { } @Test - func `missing parent equal counter siblings fail open`() throws { + func `missing parent equal counter siblings fail closed`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -174,8 +174,9 @@ struct Issue2037ScannerIntegrationTests { partial + (row.inputTokens ?? 0) + (row.cacheReadTokens ?? 0) + (row.outputTokens ?? 0) } - // Each unresolved child skips its first cumulative snapshot, then independently bills - // five input tokens. Equal token vectors are not sufficient cross-file identity. - #expect(scannedUnits == 10) + // Neither child has a trustworthy inherited baseline. Equal token vectors are not + // sufficient cross-file identity, so both children stay uncounted until the parent + // snapshot is available from its file or the persistent token index. + #expect(scannedUnits == 0) } } diff --git a/Tests/CodexBarTests/UsageStoreCachedTokenHydrationTests.swift b/Tests/CodexBarTests/UsageStoreCachedTokenHydrationTests.swift index 0237fe4542..c120e04079 100644 --- a/Tests/CodexBarTests/UsageStoreCachedTokenHydrationTests.swift +++ b/Tests/CodexBarTests/UsageStoreCachedTokenHydrationTests.swift @@ -191,6 +191,63 @@ struct UsageStoreCachedTokenHydrationTests { #expect(tokenRefreshCount == 1) } + @Test + func `incompatible cached hydration remains visible and starts marked catch-up`() async throws { + let staleAt = Date(timeIntervalSince1970: 1_775_000_000) + let settings = Self.makeCodexOnlySettings(historyDays: 1) + let store = UsageStore( + fetcher: UsageFetcher(), + browserDetection: BrowserDetection(cacheTTL: 0), + settings: settings, + startupBehavior: .testing, + environmentBase: [:]) + var observedStalePresentation = false + var statusLoadCount = 0 + store._test_cachedCodexTokenSnapshotLoaderOverride = { _, _, _ in + (Self.cachedTokenSnapshot(), nil, staleAt) + } + store._test_tokenUsageSnapshotLoaderOverride = { _, _, now, _, _ in + CostUsageTokenSnapshot( + sessionTokens: 84, + sessionCostUSD: 2, + last30DaysTokens: 84, + last30DaysCostUSD: 2, + daily: [], + updatedAt: now) + } + store._test_codexCostCatchUpStatusOverride = { _ in + statusLoadCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: statusLoadCount == 1, + progressKey: "status-\(statusLoadCount)", + staleSnapshotUpdatedAt: statusLoadCount == 1 ? staleAt : nil) + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + CostUsageFetcher.CodexScanCatchUpStatus( + pending: false, + progressKey: "complete") + } + store._test_codexCostCatchUpSleepOverride = { _ in + observedStalePresentation = + store.codexCostCatchUpActivity?.staleSnapshotUpdatedAt == staleAt + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + let hydration = store.hydrateCachedTokenSnapshots() + await hydration?.value + for _ in 0..<1000 where store.codexCostCatchUpTask != nil { + try await Task.sleep(for: .milliseconds(1)) + } + + #expect(observedStalePresentation) + #expect(store.codexCostCatchUpTask == nil) + #expect(store.codexCostCatchUpActivity?.phase == .complete) + #expect(store.codexCostCatchUpActivity?.staleSnapshotUpdatedAt == nil) + } + @Test func `confirmed empty publication wins over in flight cached codex hydration`() async { let settings = Self.makeCodexOnlySettings(historyDays: 1) @@ -203,7 +260,7 @@ struct UsageStoreCachedTokenHydrationTests { let gate = CachedTokenHydrationGate() store._test_cachedCodexTokenSnapshotLoaderOverride = { _, _, _ in await gate.enter() - return (Self.cachedTokenSnapshot(), Date()) + return (Self.cachedTokenSnapshot(), Date(), nil) } let hydration = store.hydrateCachedTokenSnapshots() diff --git a/Tests/CodexBarTests/UsageStoreCodexCostCatchUpTests.swift b/Tests/CodexBarTests/UsageStoreCodexCostCatchUpTests.swift new file mode 100644 index 0000000000..9ccd317d33 --- /dev/null +++ b/Tests/CodexBarTests/UsageStoreCodexCostCatchUpTests.swift @@ -0,0 +1,212 @@ +import Foundation +import Testing +@testable import CodexBar +@testable import CodexBarCore + +@MainActor +@Suite(.serialized) +struct UsageStoreCodexCostCatchUpTests { + @Test + func `bounded catch-up automatically publishes only the final stable snapshot`() async throws { + let store = try Self.makeStore(suite: "publishes-final") + var snapshotLoadCount = 0 + var statusLoadCount = 0 + var advanceCount = 0 + var sleepDurations: [TimeInterval] = [] + store._test_tokenUsageSnapshotLoaderOverride = { _, _, now, _, _ in + snapshotLoadCount += 1 + return Self.tokenSnapshot(cost: Double(snapshotLoadCount), now: now) + } + store._test_codexCostCatchUpStatusOverride = { _ in + statusLoadCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: statusLoadCount == 1, + progressKey: "status-\(statusLoadCount)") + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: advanceCount < 2, + progressKey: "advance-\(advanceCount)") + } + store._test_codexCostCatchUpSleepOverride = { duration in + sleepDurations.append(duration) + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + await store.refreshTokenUsage(.codex, force: true) + await Self.waitUntil { + store.codexCostCatchUpTask == nil && snapshotLoadCount == 2 + } + + #expect(advanceCount == 2) + #expect(statusLoadCount == 2) + #expect(snapshotLoadCount == 2) + #expect(sleepDurations.first == 8) + #expect(store.tokenSnapshot(for: .codex)?.last30DaysCostUSD == 2) + #expect(store.tokenSnapshotPublicationRevision(for: .codex) == 2) + #expect(store.tokenError(for: .codex) == nil) + } + + @Test + func `catch-up stops after one bounded pass that makes no progress`() async throws { + let store = try Self.makeStore(suite: "no-progress") + var snapshotLoadCount = 0 + var advanceCount = 0 + store._test_tokenUsageSnapshotLoaderOverride = { _, _, now, _, _ in + snapshotLoadCount += 1 + return Self.tokenSnapshot(cost: 1, now: now) + } + store._test_codexCostCatchUpStatusOverride = { _ in + CostUsageFetcher.CodexScanCatchUpStatus(pending: true, progressKey: "unchanged") + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus(pending: true, progressKey: "unchanged") + } + store._test_codexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + await store.refreshTokenUsage(.codex, force: true) + await Self.waitUntil { + store.codexCostCatchUpTask == nil && advanceCount == 1 + } + + #expect(advanceCount == 1) + #expect(snapshotLoadCount == 1) + #expect(store.tokenSnapshot(for: .codex)?.last30DaysCostUSD == 1) + #expect(store.tokenSnapshotPublicationRevision(for: .codex) == 1) + #expect(store.codexCostCatchUpActivity?.phase == .paused) + #expect(store.codexCostCatchUpActivity?.pauseReason == .noProgress) + } + + @Test + func `accelerated catch-up runs without an inter-pass delay and publishes progress`() async throws { + let store = try Self.makeStore(suite: "accelerated") + var statusLoadCount = 0 + var sleepDurations: [TimeInterval] = [] + store._test_tokenUsageSnapshotLoaderOverride = { _, _, now, _, _ in + Self.tokenSnapshot(cost: 1, now: now) + } + store._test_codexCostCatchUpStatusOverride = { _ in + statusLoadCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: statusLoadCount == 1, + progressKey: "status-\(statusLoadCount)", + processedBytes: statusLoadCount == 1 ? 25 : 100, + totalBytes: 100, + completedFiles: statusLoadCount == 1 ? 0 : 1, + totalFiles: 1) + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + CostUsageFetcher.CodexScanCatchUpStatus( + pending: false, + progressKey: "complete", + processedBytes: 100, + totalBytes: 100, + completedFiles: 1, + totalFiles: 1) + } + store._test_codexCostCatchUpSleepOverride = { duration in + sleepDurations.append(duration) + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.battery, true, .serious) + } + + store.startCodexCostCatchUpIfNeeded(mode: .accelerated) + await Self.waitUntil { + store.codexCostCatchUpTask == nil + } + + #expect(sleepDurations.first == 0) + #expect(store.codexCostCatchUpActivity?.phase == .complete) + #expect(store.codexCostCatchUpActivity?.mode == .accelerated) + #expect(store.codexCostCatchUpActivity?.fractionCompleted == 1) + } + + @Test + func `stop during an idle delay preserves progress without starting a pass`() async throws { + let store = try Self.makeStore(suite: "stop-idle") + var advanceCount = 0 + store._test_codexCostCatchUpStatusOverride = { _ in + CostUsageFetcher.CodexScanCatchUpStatus( + pending: true, + progressKey: "partial", + processedBytes: 50, + totalBytes: 100) + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus(pending: false, progressKey: "unexpected") + } + store._test_codexCostCatchUpSleepOverride = { _ in + store.stopCodexCostCatchUp() + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startCodexCostCatchUpIfNeeded() + await Self.waitUntil { + store.codexCostCatchUpTask == nil + } + + #expect(advanceCount == 0) + #expect(store.codexCostCatchUpActivity?.phase == .paused) + #expect(store.codexCostCatchUpActivity?.pauseReason == .user) + #expect(store.codexCostCatchUpActivity?.fractionCompleted == 0.5) + } + + private static func makeStore(suite: String) throws -> UsageStore { + let settings = testSettingsStore(suiteName: "UsageStoreCodexCostCatchUpTests-\(suite)") + settings.costUsageEnabled = true + settings.costUsageHistoryDays = 30 + let metadata = try #require(ProviderRegistry.shared.metadata[.codex]) + settings.setProviderEnabled(provider: .codex, metadata: metadata, enabled: true) + return UsageStore( + fetcher: UsageFetcher(environment: [:]), + browserDetection: BrowserDetection(cacheTTL: 0), + settings: settings, + startupBehavior: .testing, + environmentBase: [:]) + } + + private static func tokenSnapshot(cost: Double, now: Date) -> CostUsageTokenSnapshot { + CostUsageTokenSnapshot( + sessionTokens: 10, + sessionCostUSD: cost, + last30DaysTokens: 10, + last30DaysCostUSD: cost, + daily: [CostUsageDailyReport.Entry( + date: "2026-07-30", + inputTokens: 4, + outputTokens: 6, + totalTokens: 10, + costUSD: cost, + modelsUsed: nil, + modelBreakdowns: nil)], + updatedAt: now) + } + + private static func waitUntil( + _ condition: @escaping @MainActor () -> Bool) async + { + for _ in 0..<1000 { + if condition() { + return + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + Issue.record("Timed out waiting for Codex cost catch-up task") + } +} diff --git a/Tests/CodexBarTests/UsageStoreDisabledProviderCleanupTests.swift b/Tests/CodexBarTests/UsageStoreDisabledProviderCleanupTests.swift index 01a4a77473..9b208cc261 100644 --- a/Tests/CodexBarTests/UsageStoreDisabledProviderCleanupTests.swift +++ b/Tests/CodexBarTests/UsageStoreDisabledProviderCleanupTests.swift @@ -381,7 +381,8 @@ struct UsageStoreDisabledProviderCleanupTests { await gate.suspend() return ( snapshot: Self.tokenSnapshot(tokens: 710, historyDays: historyDays, updatedAt: now), - lastRefreshAt: now) + lastRefreshAt: now, + staleSnapshotUpdatedAt: nil) } store.hydrateCachedTokenSnapshots() diff --git a/Tests/CodexBarTests/UsageStoreSpendDashboardCodexCostCatchUpTests.swift b/Tests/CodexBarTests/UsageStoreSpendDashboardCodexCostCatchUpTests.swift new file mode 100644 index 0000000000..93a5419a75 --- /dev/null +++ b/Tests/CodexBarTests/UsageStoreSpendDashboardCodexCostCatchUpTests.swift @@ -0,0 +1,196 @@ +import Foundation +import Testing +@testable import CodexBar +@testable import CodexBarCore + +@MainActor +@Suite(.serialized) +struct UsageStoreSpendDashboardCodexCostCatchUpTests { + @Test + func `dashboard catch-up advances every account cache and publishes a reload revision`() async throws { + let store = try Self.makeStore(suite: "all-accounts") + let accounts = [ + Self.account(id: "first", cacheIdentity: "cache-first"), + Self.account(id: "second", cacheIdentity: "cache-second"), + ] + let baselineConfiguration = SpendDashboardSource.configuration(settings: store.settings, store: store) + var completedCacheIdentities: Set = [] + var statusAccounts: [String] = [] + var advancedAccounts: [String] = [] + var receivedHistoryDays: [Int] = [] + store._test_spendDashboardCodexCostCatchUpStatusOverride = { account in + statusAccounts.append(account.id) + let complete = completedCacheIdentities.contains(account.cacheIdentity) + return Self.status( + pending: !complete, + key: complete ? "complete-\(account.id)" : "pending-\(account.id)", + processedBytes: complete ? 100 : 25) + } + store._test_spendDashboardCodexCostCatchUpAdvanceOverride = { account, _, historyDays in + advancedAccounts.append(account.id) + receivedHistoryDays.append(historyDays) + completedCacheIdentities.insert(account.cacheIdentity) + return Self.status( + pending: false, + key: "complete-\(account.id)", + processedBytes: 100) + } + store._test_spendDashboardCodexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_spendDashboardCodexCostCatchUpResourceStateOverride = { + (.battery, true, .serious) + } + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts, mode: .accelerated) + await Self.waitUntil { + store.spendDashboardCodexCostCatchUpTask == nil + } + + let replacementConfiguration = SpendDashboardSource.configuration(settings: store.settings, store: store) + #expect(statusAccounts == ["first", "second"]) + #expect(advancedAccounts == ["first", "second"]) + #expect(receivedHistoryDays == [SpendDashboardSource.scanDays, SpendDashboardSource.scanDays]) + #expect(store.spendDashboardCodexCostCatchUpRevision == 1) + #expect(baselineConfiguration.sourceRevisions != replacementConfiguration.sourceRevisions) + #expect(store.spendDashboardCodexCostCatchUpActivity?.phase == .complete) + #expect(store.spendDashboardCodexCostCatchUpActivity?.mode == .accelerated) + #expect(store.spendDashboardCodexCostCatchUpActivity?.fractionCompleted == 1) + } + + @Test + func `a stalled account cache does not prevent a sibling cache from advancing`() async throws { + let store = try Self.makeStore(suite: "stalled-sibling") + let accounts = [ + Self.account(id: "stalled", cacheIdentity: "cache-stalled"), + Self.account(id: "healthy", cacheIdentity: "cache-healthy"), + ] + var advancedAccounts: [String] = [] + store._test_spendDashboardCodexCostCatchUpStatusOverride = { account in + Self.status( + pending: true, + key: "pending-\(account.id)", + processedBytes: 25) + } + store._test_spendDashboardCodexCostCatchUpAdvanceOverride = { account, _, _ in + advancedAccounts.append(account.id) + if account.id == "stalled" { + return Self.status( + pending: true, + key: "pending-stalled", + processedBytes: 25) + } + return Self.status( + pending: false, + key: "complete-healthy", + processedBytes: 100) + } + store._test_spendDashboardCodexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_spendDashboardCodexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts, mode: .accelerated) + await Self.waitUntil { + store.spendDashboardCodexCostCatchUpTask == nil + } + + #expect(advancedAccounts == ["stalled", "healthy"]) + #expect(store.spendDashboardCodexCostCatchUpRevision == 1) + #expect(store.spendDashboardCodexCostCatchUpActivity?.phase == .paused) + #expect(store.spendDashboardCodexCostCatchUpActivity?.pauseReason == .noProgress) + } + + @Test + func `a no-progress pass does not publish a reload revision`() async throws { + let store = try Self.makeStore(suite: "no-progress-revision") + let accounts = [Self.account(id: "stalled", cacheIdentity: "cache-stalled")] + store._test_spendDashboardCodexCostCatchUpStatusOverride = { _ in + Self.status(pending: true, key: "unchanged", processedBytes: 25) + } + store._test_spendDashboardCodexCostCatchUpAdvanceOverride = { _, _, _ in + Self.status(pending: true, key: "unchanged", processedBytes: 25) + } + store._test_spendDashboardCodexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_spendDashboardCodexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts, mode: .accelerated) + await Self.waitUntil { + store.spendDashboardCodexCostCatchUpTask == nil + } + + #expect(store.spendDashboardCodexCostCatchUpRevision == 0) + #expect(store.spendDashboardCodexCostCatchUpActivity?.phase == .paused) + #expect(store.spendDashboardCodexCostCatchUpActivity?.pauseReason == .noProgress) + } + + @Test + func `dashboard synchronization keeps an accelerated account queue accelerated`() throws { + let store = try Self.makeStore(suite: "preserve-mode") + let accounts = [Self.account(id: "account", cacheIdentity: "cache-account")] + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts, mode: .accelerated) + let originalToken = store.spendDashboardCodexCostCatchUpToken + store.synchronizeSpendDashboardCodexCostCatchUp(accounts: accounts) + + #expect(originalToken != nil) + #expect(store.spendDashboardCodexCostCatchUpToken == originalToken) + #expect(store.spendDashboardCodexCostCatchUpMode == .accelerated) + store.cancelSpendDashboardCodexCostCatchUp() + } + + private static func makeStore(suite: String) throws -> UsageStore { + let settings = testSettingsStore( + suiteName: "UsageStoreSpendDashboardCodexCostCatchUpTests-\(suite)") + settings.costUsageEnabled = true + let metadata = try #require(ProviderRegistry.shared.metadata[.codex]) + settings.setProviderEnabled(provider: .codex, metadata: metadata, enabled: true) + return UsageStore( + fetcher: UsageFetcher(environment: [:]), + browserDetection: BrowserDetection(cacheTTL: 0), + settings: settings, + startupBehavior: .testing, + environmentBase: [:]) + } + + private static func account(id: String, cacheIdentity: String) -> CodexSpendScanRequest { + CodexSpendScanRequest( + id: id, + displayName: "Codex · \(id)", + source: .profileHome(path: "/synthetic/\(id)"), + homePath: "/synthetic/\(id)", + authFingerprint: nil, + authFileWasReadable: false, + cacheIdentity: cacheIdentity) + } + + private static func status( + pending: Bool, + key: String, + processedBytes: Int64) -> CostUsageFetcher.CodexScanCatchUpStatus + { + CostUsageFetcher.CodexScanCatchUpStatus( + pending: pending, + progressKey: key, + processedBytes: processedBytes, + totalBytes: 100, + completedFiles: pending ? 0 : 1, + totalFiles: 1) + } + + private static func waitUntil(_ condition: @escaping @MainActor () -> Bool) async { + for _ in 0..<1000 { + if condition() { + return + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + Issue.record("Timed out waiting for Spend Dashboard Codex cost catch-up") + } +}