From 9eac2c18d7ef9524e0a2d82f5f17916a514c421e Mon Sep 17 00:00:00 2001 From: Junho Yeo Date: Mon, 22 Jun 2026 17:46:59 +0900 Subject: [PATCH 1/3] fix(usage): make subscription providers fail-soft Audit found several subscription-usage providers emitting confident-looking empty/zero data instead of omitting it: - amp: fetch() returned Ok with no metrics when its $X/$Y display-text scrape didn't match, rendering a bare "Amp" header. Now bails so it's dropped. Also made the byte-offset slicing char-boundary-safe (could panic on multibyte display_text). - copilot, minimax_tokenplan: same empty-metrics guard / skip empty rows. - warp: "Spend" and no-limit "requests" metrics set 0% remaining, rendering a false "exhausted" bar; now read full with the figure kept in the label. Dropped the placeholder "Aggregate API cache" plan string. - zai: a missing percentage was rendered as "100% left"; now skipped. - kimi: include resets_at in the metric dedup key so distinct windows with equal numbers don't collapse. Adds unit tests for the amp and warp behavior changes. Confidence: high Scope-risk: narrow --- crates/tokscale-cli/src/commands/usage/amp.rs | 116 +++++++++++++----- .../src/commands/usage/copilot.rs | 6 + .../tokscale-cli/src/commands/usage/kimi.rs | 10 +- .../src/commands/usage/minimax_tokenplan.rs | 7 +- .../tokscale-cli/src/commands/usage/warp.rs | 87 +++++++++++-- crates/tokscale-cli/src/commands/usage/zai.rs | 7 +- 6 files changed, 182 insertions(+), 51 deletions(-) diff --git a/crates/tokscale-cli/src/commands/usage/amp.rs b/crates/tokscale-cli/src/commands/usage/amp.rs index da601de9b..9044bef1b 100644 --- a/crates/tokscale-cli/src/commands/usage/amp.rs +++ b/crates/tokscale-cli/src/commands/usage/amp.rs @@ -41,11 +41,13 @@ fn read_credentials() -> Result { /// Parse a dollar amount like "$4.50" or "$1,200.00" from text starting at the given prefix. fn parse_dollar_after(text: &str, prefix: &str) -> Option { let start = text.find(prefix)? + prefix.len(); - let rest = &text[start..]; + // `start` is a valid byte offset (end of a found substring), but slice it + // via `get` so arbitrary display_text can never panic on a bad boundary. + let rest = text.get(start..)?; let end = rest .find(|c: char| !c.is_ascii_digit() && c != '.' && c != ',') .unwrap_or(rest.len()); - let num_str = &rest[..end]; + let num_str = rest.get(..end)?; num_str.replace(',', "").parse().ok() } @@ -55,40 +57,48 @@ fn parse_display_text(text: &str) -> Vec { // Parse free tier: "$X/$Y remaining" // Look for pattern like "$4.50/$20.00 remaining" if let Some(slash_pos) = text.find("/$") { - if let Some(dollar_before) = text[..slash_pos].rfind('$') { - let before = &text[dollar_before + 1..slash_pos]; - if let Ok(remaining) = before.replace(',', "").parse::() { - // Find the total after /$ - let after = &text[slash_pos + 2..]; - if let Some(space_pos) = after.find(|c: char| c.is_ascii_whitespace()) { - if let Ok(total) = after[..space_pos].replace(',', "").parse::() { - if total > 0.0 && total.is_finite() && remaining.is_finite() { - let used = (total - remaining).max(0.0); - let used_pct = if used.is_finite() { - (used / total * 100.0).clamp(0.0, 100.0) - } else { - 0.0 - }; - let remaining_pct = (100.0 - used_pct).clamp(0.0, 100.0); - let mut resets_at = None; - - // Estimate reset time from hourly replenish rate - if let Some(rate) = parse_dollar_after(text, "+$") { - if rate > 0.0 && used > 0.0 && rate.is_finite() { - let secs = (used / rate * 3600.0) as i64; - let resets = - chrono::Utc::now() + chrono::Duration::seconds(secs); - resets_at = Some(resets.to_rfc3339()); + if let Some(dollar_before) = text.get(..slash_pos).and_then(|s| s.rfind('$')) { + // All byte offsets below come from `find`/`rfind` on `text`, so they + // sit on char boundaries; `get` keeps it panic-free regardless. + if let Some(before) = text.get(dollar_before + 1..slash_pos) { + if let Ok(remaining) = before.replace(',', "").parse::() { + // Find the total after /$ + if let Some(after) = text.get(slash_pos + 2..) { + // `space_pos` comes from `after.find`, so `after[..space_pos]` + // is always on a char boundary. + if let Some(space_pos) = after.find(|c: char| c.is_ascii_whitespace()) { + if let Ok(total) = after[..space_pos].replace(',', "").parse::() { + if total > 0.0 && total.is_finite() && remaining.is_finite() { + let used = (total - remaining).max(0.0); + let used_pct = if used.is_finite() { + (used / total * 100.0).clamp(0.0, 100.0) + } else { + 0.0 + }; + let remaining_pct = (100.0 - used_pct).clamp(0.0, 100.0); + let mut resets_at = None; + + // Estimate reset time from hourly replenish rate + if let Some(rate) = parse_dollar_after(text, "+$") { + if rate > 0.0 && used > 0.0 && rate.is_finite() { + let secs = (used / rate * 3600.0) as i64; + let resets = chrono::Utc::now() + + chrono::Duration::seconds(secs); + resets_at = Some(resets.to_rfc3339()); + } + } + + metrics.push(UsageMetric { + label: "Free".into(), + used_percent: used_pct, + remaining_percent: remaining_pct, + remaining_label: Some(format!( + "${remaining:.2}/${total:.2}" + )), + resets_at, + }); } } - - metrics.push(UsageMetric { - label: "Free".into(), - used_percent: used_pct, - remaining_percent: remaining_pct, - remaining_label: Some(format!("${remaining:.2}/${total:.2}")), - resets_at, - }); } } } @@ -164,6 +174,9 @@ pub fn fetch() -> Result { let display_text = body.result.and_then(|r| r.display_text).unwrap_or_default(); let metrics = parse_display_text(&display_text); + if metrics.is_empty() { + anyhow::bail!("Amp returned no parseable usage (display_text format may have changed)"); + } let plan = detect_plan(&metrics); Ok(UsageOutput { @@ -178,3 +191,38 @@ pub fn fetch() -> Result { }) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_free_tier_balance() { + let metrics = parse_display_text("$4.50/$20.00 remaining"); + assert_eq!(metrics.len(), 1); + assert_eq!(metrics[0].label, "Free"); + // $4.50 remaining of $20.00 -> 77.5% used, 22.5% left. + assert!((metrics[0].used_percent - 77.5).abs() < 1e-9); + assert!((metrics[0].remaining_percent - 22.5).abs() < 1e-9); + } + + #[test] + fn empty_display_text_yields_no_metrics() { + // fetch() bails on an empty metrics set so fetch_all drops the provider + // instead of rendering a bare header row. + assert!(parse_display_text("").is_empty()); + assert!(parse_display_text("no dollar figures here").is_empty()); + } + + #[test] + fn multibyte_display_text_does_not_panic() { + // Byte offsets from find/rfind must stay on char boundaries; arbitrary + // UTF-8 around the markers must never panic. + let _ = parse_display_text("残高 $4.50/$20.00 残り 한국어"); + let _ = parse_display_text("Individual credits: $5.00 残り"); + let _ = parse_dollar_after("プレフィックス€$1.23é", "€$"); + let _ = parse_dollar_after("+$0.50円/hr", "+$"); + // Marker present but followed immediately by a multibyte char. + let _ = parse_dollar_after("/$é", "/$"); + } +} diff --git a/crates/tokscale-cli/src/commands/usage/copilot.rs b/crates/tokscale-cli/src/commands/usage/copilot.rs index 41f247efc..995f82464 100644 --- a/crates/tokscale-cli/src/commands/usage/copilot.rs +++ b/crates/tokscale-cli/src/commands/usage/copilot.rs @@ -278,6 +278,12 @@ pub fn fetch() -> Result { } } + if metrics.is_empty() { + anyhow::bail!( + "Copilot returned no parseable usage (quota response format may have changed)" + ); + } + Ok(UsageOutput { provider: "Copilot".into(), account: None, diff --git a/crates/tokscale-cli/src/commands/usage/kimi.rs b/crates/tokscale-cli/src/commands/usage/kimi.rs index 9940e65ec..0d0da06b3 100644 --- a/crates/tokscale-cli/src/commands/usage/kimi.rs +++ b/crates/tokscale-cli/src/commands/usage/kimi.rs @@ -256,10 +256,11 @@ pub fn fetch() -> Result { }; if let Some(metric) = parse_quota_detail(label, detail) { let key = format!( - "{}:{}:{}", + "{}:{}:{}:{}", label, metric.used_percent, - metric.remaining_label.as_deref().unwrap_or("") + metric.remaining_label.as_deref().unwrap_or(""), + metric.resets_at.as_deref().unwrap_or("") ); if seen.insert(key) { metrics.push(metric); @@ -273,10 +274,11 @@ pub fn fetch() -> Result { if let Some(ref usage) = resp.usage { if let Some(metric) = parse_quota_detail("Weekly", usage) { let key = format!( - "{}:{}:{}", + "{}:{}:{}:{}", "Weekly", metric.used_percent, - metric.remaining_label.as_deref().unwrap_or("") + metric.remaining_label.as_deref().unwrap_or(""), + metric.resets_at.as_deref().unwrap_or("") ); if seen.insert(key) { metrics.push(metric); diff --git a/crates/tokscale-cli/src/commands/usage/minimax_tokenplan.rs b/crates/tokscale-cli/src/commands/usage/minimax_tokenplan.rs index 610bd1393..95127cf56 100644 --- a/crates/tokscale-cli/src/commands/usage/minimax_tokenplan.rs +++ b/crates/tokscale-cli/src/commands/usage/minimax_tokenplan.rs @@ -190,6 +190,11 @@ pub fn fetch_all() -> Result> { } let remains = resp.model_remains.as_deref().unwrap_or(&[]); + let metrics = build_metrics(remains); + // Skip sites with no renderable windows so we don't emit a bare header row. + if metrics.is_empty() { + continue; + } outputs.push(UsageOutput { provider: "MiniMax Token Plan".into(), account: Some(UsageAccount { @@ -199,7 +204,7 @@ pub fn fetch_all() -> Result> { }), plan: None, email: None, - metrics: build_metrics(remains), + metrics, reset_credits: None, credit_status: None, spend_control: None, diff --git a/crates/tokscale-cli/src/commands/usage/warp.rs b/crates/tokscale-cli/src/commands/usage/warp.rs index 035fa0be1..a3fd8af28 100644 --- a/crates/tokscale-cli/src/commands/usage/warp.rs +++ b/crates/tokscale-cli/src/commands/usage/warp.rs @@ -5,14 +5,12 @@ pub fn has_credentials() -> bool { crate::warp::load_usage_cache().is_some() } -pub fn fetch() -> Result { - let cache = crate::warp::load_usage_cache() - .ok_or_else(|| anyhow::anyhow!("Warp aggregate usage cache not found"))?; +fn build_metrics(usage: &crate::warp::WarpAggregateUsage) -> Vec { let mut metrics = Vec::new(); - if let Some(used) = cache.usage.requests_used { + if let Some(used) = usage.requests_used { let (used_percent, remaining_percent, remaining_label) = - if let Some(limit) = cache.usage.request_limit.filter(|limit| *limit > 0) { + if let Some(limit) = usage.request_limit.filter(|limit| *limit > 0) { let used_percent = (used as f64 / limit as f64 * 100.0).clamp(0.0, 100.0); let remaining = limit.saturating_sub(used); ( @@ -21,31 +19,44 @@ pub fn fetch() -> Result { Some(format!("{remaining} requests left")), ) } else { - (0.0, 0.0, Some(format!("{used} requests used"))) + // No request limit: this is an informational counter, not a + // capped quota. Render the bar as full (100% remaining) rather + // than an empty "exhausted" bar. + (0.0, 100.0, Some(format!("{used} requests used"))) }; metrics.push(UsageMetric { label: "Requests".to_string(), used_percent, remaining_percent, remaining_label, - resets_at: cache.usage.next_refresh_time.clone(), + resets_at: usage.next_refresh_time.clone(), }); } - if let Some(spend_cents) = cache.usage.spend_cents { + if let Some(spend_cents) = usage.spend_cents { metrics.push(UsageMetric { label: "Spend".to_string(), used_percent: 0.0, - remaining_percent: 0.0, + // Spend is an informational dollar figure, not a consumed quota, so + // keep the bar full instead of rendering a false "exhausted" bar. + remaining_percent: 100.0, remaining_label: Some(format!("${:.2}", spend_cents as f64 / 100.0)), - resets_at: cache.usage.next_refresh_time.clone(), + resets_at: usage.next_refresh_time.clone(), }); } + metrics +} + +pub fn fetch() -> Result { + let cache = crate::warp::load_usage_cache() + .ok_or_else(|| anyhow::anyhow!("Warp aggregate usage cache not found"))?; + let metrics = build_metrics(&cache.usage); + Ok(UsageOutput { provider: "Warp/Oz".to_string(), account: None, - plan: Some("Aggregate API cache".to_string()), + plan: None, email: None, metrics, reset_credits: None, @@ -53,3 +64,57 @@ pub fn fetch() -> Result { spend_control: None, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::warp::WarpAggregateUsage; + + #[test] + fn spend_metric_reads_full_not_exhausted() { + let usage = WarpAggregateUsage { + spend_cents: Some(1234), + ..Default::default() + }; + let metrics = build_metrics(&usage); + let spend = metrics.iter().find(|m| m.label == "Spend").unwrap(); + // Informational $ figure: bar must read full, not a false 0%-remaining. + assert_eq!(spend.remaining_percent, 100.0); + assert_eq!(spend.used_percent, 0.0); + assert_eq!(spend.remaining_label.as_deref(), Some("$12.34")); + } + + #[test] + fn unlimited_requests_read_full_not_exhausted() { + let usage = WarpAggregateUsage { + requests_used: Some(42), + request_limit: None, + ..Default::default() + }; + let metrics = build_metrics(&usage); + let requests = metrics.iter().find(|m| m.label == "Requests").unwrap(); + assert_eq!(requests.remaining_percent, 100.0); + assert_eq!(requests.used_percent, 0.0); + assert_eq!( + requests.remaining_label.as_deref(), + Some("42 requests used") + ); + } + + #[test] + fn limited_requests_compute_usage() { + let usage = WarpAggregateUsage { + requests_used: Some(25), + request_limit: Some(100), + ..Default::default() + }; + let metrics = build_metrics(&usage); + let requests = metrics.iter().find(|m| m.label == "Requests").unwrap(); + assert_eq!(requests.used_percent, 25.0); + assert_eq!(requests.remaining_percent, 75.0); + assert_eq!( + requests.remaining_label.as_deref(), + Some("75 requests left") + ); + } +} diff --git a/crates/tokscale-cli/src/commands/usage/zai.rs b/crates/tokscale-cli/src/commands/usage/zai.rs index b8c92f55a..c957a577e 100644 --- a/crates/tokscale-cli/src/commands/usage/zai.rs +++ b/crates/tokscale-cli/src/commands/usage/zai.rs @@ -104,7 +104,12 @@ pub fn fetch() -> Result { if let Some(limits) = quota.data.as_ref().and_then(|d| d.limits.as_ref()) { for limit in limits.iter() { - let pct = limit.percentage.unwrap_or(0.0).clamp(0.0, 100.0); + // Skip limits with no percentage rather than fabricating + // "0% used / 100% left" from a missing field. + let pct = match limit.percentage { + Some(p) => p.clamp(0.0, 100.0), + None => continue, + }; match limit.limit_type.as_deref() { Some("TOKENS_LIMIT") => { From 56decd0fce3edcbc925e5f39e6140f4706aee01a Mon Sep 17 00:00:00 2001 From: Junho Yeo Date: Mon, 22 Jun 2026 17:46:59 +0900 Subject: [PATCH 2/3] fix(sessions): canonicalize providers and harden token parsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-parser audit found inconsistent provider attribution and silent data loss: - Provider canonicalization (opencode, micode, kilo, mux, antigravity, jcode): these stored a raw provider field verbatim, so aliases (fireworks, vertex, gemini, azure) split into separate aggregation buckets. Now routed through provider_identity::canonical_provider like the other parsers. - codex: provider defaulted to "openai" when session meta was absent; now infers from the model first, keeping "openai" only as the final fallback. - gjc, pi: dropped messages with valid tokens but no provider field; now recover via model inference (fallback "gjc"/"pi") instead of discarding the spend. - droid: dropped a whole session when its timestamp resolved to 0; now falls back to file mtime. - qwen, mux: had no dedup key (double-count risk on re-parse); now keyed stably. - utils: seconds->ms conversion uses saturating_mul; kiro/goose timestamp paths hardened against overflow. - kiro (estimated tokens), goose (heuristic reasoning), crush (cost-only, zero tokens): documented these source limitations so 0/estimated values aren't mistaken for bugs. Updates tests that encoded the OLD behavior: fireworks->fireworks_ai canonicalization, and gjc now keeping a previously-dropped record. Confidence: high Scope-risk: moderate Directive: roocode/cline/kilocode intentionally preserve raw apiProtocol — do NOT canonicalize them --- crates/tokscale-cli/src/tui/data/mod.rs | 6 +++-- crates/tokscale-core/src/lib.rs | 11 ++++++--- .../tokscale-core/src/sessions/antigravity.rs | 1 + crates/tokscale-core/src/sessions/codex.rs | 11 +++++++-- crates/tokscale-core/src/sessions/crush.rs | 7 ++++++ crates/tokscale-core/src/sessions/droid.rs | 23 +++++------------- crates/tokscale-core/src/sessions/gjc.rs | 24 ++++++++++++++----- crates/tokscale-core/src/sessions/goose.rs | 14 ++++++++++- crates/tokscale-core/src/sessions/jcode.rs | 7 +++--- crates/tokscale-core/src/sessions/kilo.rs | 1 + crates/tokscale-core/src/sessions/kiro.rs | 21 +++++++++++++++- crates/tokscale-core/src/sessions/micode.rs | 4 +++- crates/tokscale-core/src/sessions/mux.rs | 13 +++++++--- crates/tokscale-core/src/sessions/opencode.rs | 9 +++++-- crates/tokscale-core/src/sessions/pi.rs | 8 ++++++- crates/tokscale-core/src/sessions/qwen.rs | 9 ++++++- crates/tokscale-core/src/sessions/utils.rs | 8 +++++-- crates/tokscale-core/src/sessions/zed.rs | 4 ++-- 18 files changed, 134 insertions(+), 47 deletions(-) diff --git a/crates/tokscale-cli/src/tui/data/mod.rs b/crates/tokscale-cli/src/tui/data/mod.rs index f8273fa48..bbefb4333 100644 --- a/crates/tokscale-cli/src/tui/data/mod.rs +++ b/crates/tokscale-cli/src/tui/data/mod.rs @@ -2456,7 +2456,7 @@ after"#, let expected_cost = expected_message_cost( &pricing, "accounts/fireworks/models/deepseek-v3-0324", - "fireworks", + "fireworks_ai", CoreTokenBreakdown { input: 10, output: 5, @@ -2468,7 +2468,9 @@ after"#, assert_eq!(usage.models.len(), 1); assert_eq!(usage.models[0].client, "opencode"); - assert_eq!(usage.models[0].provider, "fireworks"); + // opencode now canonicalizes the provider (fireworks -> fireworks_ai), + // matching every other session parser. + assert_eq!(usage.models[0].provider, "fireworks_ai"); assert_eq!(usage.models[0].model, "deepseek-v3-0324"); assert_eq!(usage.models[0].tokens.total(), 15); assert_cost_matches(usage.models[0].cost, expected_cost); diff --git a/crates/tokscale-core/src/lib.rs b/crates/tokscale-core/src/lib.rs index a5fa93b28..a67f01f0c 100644 --- a/crates/tokscale-core/src/lib.rs +++ b/crates/tokscale-core/src/lib.rs @@ -6251,7 +6251,10 @@ mod tests { assert_eq!(parsed.messages.len(), 1); assert_eq!(parsed.messages[0].client, "opencode"); assert_eq!(parsed.messages[0].model_id, "deepseek-v3-0324"); - assert_eq!(parsed.messages[0].provider_id, "fireworks"); + // opencode now canonicalizes the provider segment like every other + // session parser, so the raw "fireworks" gateway id resolves to its + // canonical "fireworks_ai" tag. + assert_eq!(parsed.messages[0].provider_id, "fireworks_ai"); } #[test] @@ -6281,7 +6284,8 @@ mod tests { ); assert_eq!(messages[0].client, "opencode"); assert_eq!(messages[0].model_id, "deepseek-v3-0324"); - assert_eq!(messages[0].provider_id, "fireworks"); + // Provider is canonicalized by the opencode parser (fireworks -> fireworks_ai). + assert_eq!(messages[0].provider_id, "fireworks_ai"); } #[test] @@ -6315,7 +6319,8 @@ mod tests { ); assert_eq!(parsed.messages[0].client, "opencode"); assert_eq!(parsed.messages[0].model_id, "deepseek-v3-0324"); - assert_eq!(parsed.messages[0].provider_id, "fireworks"); + // Provider is canonicalized by the opencode parser (fireworks -> fireworks_ai). + assert_eq!(parsed.messages[0].provider_id, "fireworks_ai"); } #[test] diff --git a/crates/tokscale-core/src/sessions/antigravity.rs b/crates/tokscale-core/src/sessions/antigravity.rs index a5135de75..0da4701ed 100644 --- a/crates/tokscale-core/src/sessions/antigravity.rs +++ b/crates/tokscale-core/src/sessions/antigravity.rs @@ -69,6 +69,7 @@ fn parse_usage_row(value: &Value, fallback_model: Option<&str>) -> Option( state.session_agent.clone() }; - let provider = state.session_provider.as_deref().unwrap_or("openai"); + let provider = state + .session_provider + .as_deref() + .or_else(|| model.as_deref().and_then(inferred_provider_from_model)) + .unwrap_or("openai"); let mut message = UnifiedMessage::new_with_agent( "codex", @@ -1034,7 +1039,9 @@ fn parse_codex_headless_line( return None; } - let provider = session_provider.unwrap_or("openai"); + let provider = session_provider + .or_else(|| inferred_provider_from_model(&model)) + .unwrap_or("openai"); let agent = if session_is_headless { Some("headless".to_string()) } else { diff --git a/crates/tokscale-core/src/sessions/crush.rs b/crates/tokscale-core/src/sessions/crush.rs index a03f22c68..c40175d01 100644 --- a/crates/tokscale-core/src/sessions/crush.rs +++ b/crates/tokscale-core/src/sessions/crush.rs @@ -3,6 +3,13 @@ //! Crush persists usage in a per-project SQLite database (`crush.db`). //! The database exposes reliable session-level cost, but not reliable //! per-message token accounting for import. +//! +//! IMPORTANT: Crush is COST-ONLY. This parser intentionally emits ZERO token +//! counts (`TokenBreakdown::default()`) for every message and instead +//! distributes the reliable session-level cost across day buckets. There are +//! no trustworthy per-message token columns to populate, so a token-count +//! report showing 0 tokens for crush is EXPECTED behavior, NOT a bug — the +//! signal Crush provides is cost, not tokens. use super::utils::open_readonly_sqlite; use super::UnifiedMessage; diff --git a/crates/tokscale-core/src/sessions/droid.rs b/crates/tokscale-core/src/sessions/droid.rs index 4cd2be3ac..811068ef4 100644 --- a/crates/tokscale-core/src/sessions/droid.rs +++ b/crates/tokscale-core/src/sessions/droid.rs @@ -2,7 +2,7 @@ //! //! Parses JSON files from ~/.factory/sessions/ -use super::utils::read_file_or_none; +use super::utils::{file_modified_timestamp_ms, read_file_or_none}; use super::UnifiedMessage; use crate::{provider_identity, TokenBreakdown}; use serde::Deserialize; @@ -192,26 +192,15 @@ pub fn parse_droid_file(path: &Path) -> Vec { } }; - // Get timestamp from providerLockTimestamp or file mtime + // Get timestamp from providerLockTimestamp, falling back to file mtime + // (which itself falls back to now()). Never drop a record with real token + // usage just because the timestamp could not be resolved. let timestamp = settings .provider_lock_timestamp .and_then(|ts| chrono::DateTime::parse_from_rfc3339(&ts).ok()) .map(|dt| dt.timestamp_millis()) - .or_else(|| { - std::fs::metadata(path) - .ok() - .and_then(|m| m.modified().ok()) - .map(|t| { - t.duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0) - }) - }) - .unwrap_or(0); - - if timestamp == 0 { - return Vec::new(); - } + .filter(|&ts| ts != 0) + .unwrap_or_else(|| file_modified_timestamp_ms(path)); vec![UnifiedMessage::new( "droid", diff --git a/crates/tokscale-core/src/sessions/gjc.rs b/crates/tokscale-core/src/sessions/gjc.rs index d23eb29e2..d925f4f75 100644 --- a/crates/tokscale-core/src/sessions/gjc.rs +++ b/crates/tokscale-core/src/sessions/gjc.rs @@ -22,6 +22,7 @@ use super::utils::file_modified_timestamp_ms; use super::{normalize_workspace_key, workspace_label_from_key, UnifiedMessage}; +use crate::provider_identity::inferred_provider_from_model; use crate::TokenBreakdown; use serde::Deserialize; use std::io::{BufRead, BufReader}; @@ -170,9 +171,14 @@ pub fn parse_gjc_file(path: &Path) -> Vec { None => continue, }; + // A missing provider field is recoverable: infer it from the model name + // (and fall back to "gjc") rather than dropping a message that carries + // valid tokens. let provider = match message.provider { Some(p) => p, - None => continue, + None => inferred_provider_from_model(&model) + .unwrap_or("gjc") + .to_string(), }; // Prefer unix-ms message timestamp; fall back to entry ISO timestamp, @@ -428,7 +434,8 @@ not valid json at all } /// (d) Message missing model -> skipped, no panic. - /// Message missing provider -> skipped, no panic. + /// Message missing provider -> KEPT with an inferred/fallback provider + /// (a missing provider is recoverable, not a drop condition). /// Message missing usage -> skipped, no panic. #[test] fn test_adv_missing_model_provider_usage_skipped_no_panic() { @@ -441,14 +448,19 @@ not valid json at all "missing model should be skipped" ); - // missing provider + // missing provider: recoverable. The model "m" yields no inferred + // provider, so the parser falls back to "gjc" and keeps the message + // rather than dropping its valid tokens. let content_no_provider = r#"{"type":"session","id":"gjc_adv_d2","cwd":"/tmp"} {"type":"message","id":"no_prov","message":{"role":"assistant","model":"m","timestamp":1700000001000,"usage":{"input":1,"output":1,"cost":{"total":0.001}}}}"#; let file = create_test_file(content_no_provider); - assert!( - parse_gjc_file(file.path()).is_empty(), - "missing provider should be skipped" + let no_provider_messages = parse_gjc_file(file.path()); + assert_eq!( + no_provider_messages.len(), + 1, + "missing provider should be recovered, not dropped" ); + assert_eq!(no_provider_messages[0].provider_id, "gjc"); // missing usage let content_no_usage = r#"{"type":"session","id":"gjc_adv_d3","cwd":"/tmp"} diff --git a/crates/tokscale-core/src/sessions/goose.rs b/crates/tokscale-core/src/sessions/goose.rs index 2c43cdfdd..5c85935dd 100644 --- a/crates/tokscale-core/src/sessions/goose.rs +++ b/crates/tokscale-core/src/sessions/goose.rs @@ -33,7 +33,15 @@ fn timestamp_secs_to_ms(timestamp: f64) -> i64 { if timestamp > 1e12 { timestamp as i64 } else { - (timestamp * 1000.0) as i64 + // Seconds -> milliseconds. Scale in f64 to keep sub-second precision, + // then clamp into i64 range so a garbage/huge timestamp saturates + // rather than producing an undefined cast during the conversion. + let millis = timestamp * 1000.0; + if millis.is_nan() { + 0 + } else { + millis.clamp(i64::MIN as f64, i64::MAX as f64) as i64 + } } } @@ -190,6 +198,10 @@ pub fn parse_goose_sqlite(db_path: &Path) -> Vec { output, cache_read: 0, cache_write: 0, + // INFERRED, not a real field: Goose's schema has no reasoning + // token column. We heuristically attribute any gap between the + // reported total and (input + output) to reasoning. This is a + // best-effort estimate, not a measured count. reasoning: if total > input + output { (total - input - output).max(0) } else { diff --git a/crates/tokscale-core/src/sessions/jcode.rs b/crates/tokscale-core/src/sessions/jcode.rs index 576f982b8..62edb8e08 100644 --- a/crates/tokscale-core/src/sessions/jcode.rs +++ b/crates/tokscale-core/src/sessions/jcode.rs @@ -6,7 +6,7 @@ use super::utils::{file_modified_timestamp_ms, parse_timestamp_str}; use super::{normalize_workspace_key, workspace_label_from_key, UnifiedMessage}; -use crate::TokenBreakdown; +use crate::{provider_identity, TokenBreakdown}; use serde::Deserialize; use std::path::Path; @@ -54,11 +54,12 @@ struct JcodeTokenUsage { fn provider_id(provider_key: Option<&str>) -> String { let provider = provider_key.unwrap_or("jcode").trim(); - if provider.is_empty() { + let provider = if provider.is_empty() { "jcode".to_string() } else { provider.to_string() - } + }; + provider_identity::canonical_provider(&provider).unwrap_or(provider) } fn model_id(model: Option<&str>) -> String { diff --git a/crates/tokscale-core/src/sessions/kilo.rs b/crates/tokscale-core/src/sessions/kilo.rs index 721b325c5..33af1ca9b 100644 --- a/crates/tokscale-core/src/sessions/kilo.rs +++ b/crates/tokscale-core/src/sessions/kilo.rs @@ -129,6 +129,7 @@ pub fn parse_kilo_sqlite_with_fallback( .or_else(|| provider_identity::inferred_provider_from_model(&model_id)) .unwrap_or("kilo") .to_string(); + let provider = provider_identity::canonical_provider(&provider).unwrap_or(provider); let mut unified = UnifiedMessage::new_with_agent( "kilo", diff --git a/crates/tokscale-core/src/sessions/kiro.rs b/crates/tokscale-core/src/sessions/kiro.rs index 5352ee46d..85af039dc 100644 --- a/crates/tokscale-core/src/sessions/kiro.rs +++ b/crates/tokscale-core/src/sessions/kiro.rs @@ -220,6 +220,11 @@ pub fn parse_kiro_file(path: &Path) -> Vec { } } + // NOTE: when explicit per-turn counts are absent (the common case — + // Kiro currently reports zero), input/output below are ESTIMATED, not + // measured: input is derived from context_usage_percentage * + // context_window and output from char_count / 4. Downstream must not + // treat these as exact token counts. let explicit_input = turn.input_token_count.unwrap_or(0).max(0); let explicit_output = turn.output_token_count.unwrap_or(0).max(0); let input = if explicit_input > 0 { @@ -291,7 +296,16 @@ fn estimate_tokens(chars: usize) -> i64 { } fn seconds_to_millis(seconds: f64) -> i64 { - (seconds * 1000.0) as i64 + // Scale fractional seconds to milliseconds (preserving sub-second + // precision), then clamp into i64 range. The `f64 as i64` cast saturates + // rather than wrapping on out-of-range/garbage timestamps, so the + // seconds->ms conversion cannot overflow. + let millis = seconds * 1000.0; + if millis.is_nan() { + 0 + } else { + millis.clamp(i64::MIN as f64, i64::MAX as f64) as i64 + } } fn duration_between_ms(start_ms: Option, end_ms: Option) -> Option { @@ -601,6 +615,11 @@ pub fn parse_kiro_sqlite(db_path: &Path) -> Vec { continue; }; + // NOTE: these are ESTIMATED, not measured token counts. Kiro's + // conversations_v2 does not record real per-turn token usage, so + // input is derived from context_usage_percentage * context_window + // and output from response_size (char_count) / 4. Downstream must + // not treat these as exact. let ctx_pct = meta.context_usage_percentage.unwrap_or(0.0); let response_size = meta.response_size.unwrap_or(0); diff --git a/crates/tokscale-core/src/sessions/micode.rs b/crates/tokscale-core/src/sessions/micode.rs index c54d9db75..836fc68b1 100644 --- a/crates/tokscale-core/src/sessions/micode.rs +++ b/crates/tokscale-core/src/sessions/micode.rs @@ -8,7 +8,7 @@ use super::{ normalize_opencode_agent_name, normalize_workspace_key, workspace_label_from_key, UnifiedMessage, }; -use crate::TokenBreakdown; +use crate::{provider_identity, TokenBreakdown}; use serde::Deserialize; use std::collections::HashMap; use std::path::Path; @@ -227,6 +227,8 @@ pub fn parse_micode_sqlite(db_path: &Path) -> Vec { }; let provider_id = msg.provider_id.unwrap_or_else(|| "unknown".to_string()); + let provider_id = + provider_identity::canonical_provider(&provider_id).unwrap_or(provider_id); let agent_or_mode = msg.mode.or(msg.agent); let agent = agent_or_mode.map(|a| normalize_opencode_agent_name(&a)); let input = tokens.input.max(0); diff --git a/crates/tokscale-core/src/sessions/mux.rs b/crates/tokscale-core/src/sessions/mux.rs index 4f74d591a..fd50dcb9d 100644 --- a/crates/tokscale-core/src/sessions/mux.rs +++ b/crates/tokscale-core/src/sessions/mux.rs @@ -4,7 +4,7 @@ use super::utils::{file_modified_timestamp_ms, read_file_or_none}; use super::UnifiedMessage; -use crate::TokenBreakdown; +use crate::{provider_identity, TokenBreakdown}; use serde::Deserialize; use std::collections::HashMap; use std::path::Path; @@ -73,7 +73,8 @@ pub fn parse_mux_file(path: &Path) -> Vec { by_model .into_iter() - .filter_map(|(model_key, model_usage)| { + .enumerate() + .filter_map(|(index, (model_key, model_usage))| { let tokens = |b: &Option| b.as_ref().and_then(|b| b.tokens).unwrap_or(0).max(0); let cost = @@ -93,6 +94,10 @@ pub fn parse_mux_file(path: &Path) -> Vec { return None; } + // Stable per-row dedup key so incremental re-parse collapses the + // same model entry instead of double-counting it. + let dedup_key = Some(format!("mux:{model_key}:{index}")); + // Strip "provider:" prefix for model ID (e.g., "anthropic:claude-opus-4-6" -> "claude-opus-4-6") let (provider, model_id) = if model_key.contains(':') { let mut parts = model_key.splitn(2, ':'); @@ -102,8 +107,9 @@ pub fn parse_mux_file(path: &Path) -> Vec { } else { (String::new(), model_key) }; + let provider = provider_identity::canonical_provider(&provider).unwrap_or(provider); - Some(UnifiedMessage::new( + Some(UnifiedMessage::new_with_dedup( "mux", model_id, provider, @@ -117,6 +123,7 @@ pub fn parse_mux_file(path: &Path) -> Vec { reasoning, }, source_cost, + dedup_key, )) }) .collect() diff --git a/crates/tokscale-core/src/sessions/opencode.rs b/crates/tokscale-core/src/sessions/opencode.rs index 95c754460..c3ab8ca66 100644 --- a/crates/tokscale-core/src/sessions/opencode.rs +++ b/crates/tokscale-core/src/sessions/opencode.rs @@ -9,7 +9,7 @@ use super::{ normalize_opencode_agent_name, normalize_workspace_key, workspace_label_from_key, UnifiedMessage, }; -use crate::TokenBreakdown; +use crate::{provider_identity, TokenBreakdown}; #[cfg(test)] use rusqlite::Connection; use serde::{Deserialize, Serialize}; @@ -167,10 +167,13 @@ pub fn parse_opencode_file(path: &Path) -> Option { .map(|s| s.to_string()) }); + let provider_id = msg.provider_id.unwrap_or_else(|| "unknown".to_string()); + let provider_id = provider_identity::canonical_provider(&provider_id).unwrap_or(provider_id); + let mut unified = UnifiedMessage::new_with_agent( "opencode", model_id, - msg.provider_id.unwrap_or_else(|| "unknown".to_string()), + provider_id, session_id, msg.time.created as i64, TokenBreakdown { @@ -268,6 +271,8 @@ pub fn parse_opencode_sqlite(db_path: &Path) -> Vec { }; let provider_id = msg.provider_id.unwrap_or_else(|| "unknown".to_string()); + let provider_id = + provider_identity::canonical_provider(&provider_id).unwrap_or(provider_id); let agent_or_mode = msg.mode.or(msg.agent); let agent = agent_or_mode.map(|a| normalize_opencode_agent_name(&a)); let input = tokens.input.max(0); diff --git a/crates/tokscale-core/src/sessions/pi.rs b/crates/tokscale-core/src/sessions/pi.rs index 6d9185199..b3f192227 100644 --- a/crates/tokscale-core/src/sessions/pi.rs +++ b/crates/tokscale-core/src/sessions/pi.rs @@ -4,6 +4,7 @@ use super::utils::file_modified_timestamp_ms; use super::{normalize_workspace_key, workspace_label_from_key, UnifiedMessage}; +use crate::provider_identity::inferred_provider_from_model; use crate::TokenBreakdown; use serde::Deserialize; use std::io::{BufRead, BufReader}; @@ -128,9 +129,14 @@ pub fn parse_pi_file(path: &Path) -> Vec { None => continue, }; + // A missing provider field is recoverable: infer it from the model name + // (and fall back to "pi") rather than dropping a message that carries + // valid tokens. let provider = match message.provider { Some(p) => p, - None => continue, + None => inferred_provider_from_model(&model) + .unwrap_or("pi") + .to_string(), }; let timestamp = entry diff --git a/crates/tokscale-core/src/sessions/qwen.rs b/crates/tokscale-core/src/sessions/qwen.rs index 79e759b32..b56072840 100644 --- a/crates/tokscale-core/src/sessions/qwen.rs +++ b/crates/tokscale-core/src/sessions/qwen.rs @@ -84,6 +84,9 @@ pub fn parse_qwen_file(path: &Path) -> Vec { let reader = BufReader::new(file); let mut messages: Vec = Vec::new(); + // Qwen JSONL lines carry no per-message id, so anchor the dedup key to the + // stable position of the emitted message within its session. + let mut message_index: usize = 0; for line in reader.lines() { let line = match line { @@ -137,7 +140,10 @@ pub fn parse_qwen_file(path: &Path) -> Vec { let line_session_id = extract_session_id_with_fallback(path, qwen_line.session_id.as_deref()); - let mut unified = UnifiedMessage::new( + let dedup_key = Some(format!("qwen:{line_session_id}:{message_index}")); + message_index += 1; + + let mut unified = UnifiedMessage::new_with_dedup( "qwen", model, DEFAULT_PROVIDER, @@ -151,6 +157,7 @@ pub fn parse_qwen_file(path: &Path) -> Vec { reasoning, }, 0.0, // Cost calculated later by pricing resolver + dedup_key, ); unified.set_workspace(workspace_key.clone(), workspace_label.clone()); messages.push(unified); diff --git a/crates/tokscale-core/src/sessions/utils.rs b/crates/tokscale-core/src/sessions/utils.rs index 7c89678e3..4a3c0e236 100644 --- a/crates/tokscale-core/src/sessions/utils.rs +++ b/crates/tokscale-core/src/sessions/utils.rs @@ -31,7 +31,9 @@ pub(crate) fn parse_timestamp_value(value: &Value) -> Option { if numeric >= 1_000_000_000_000 { Some(numeric) } else { - Some(numeric * 1000) + // Seconds -> milliseconds: saturating so a garbage/huge timestamp + // cannot overflow i64 during the conversion. + Some(numeric.saturating_mul(1000)) } } @@ -47,7 +49,9 @@ pub(crate) fn parse_timestamp_str(value: &str) -> Option { if numeric >= 1_000_000_000_000 { return Some(numeric); } - return Some(numeric * 1000); + // Seconds -> milliseconds: saturating so a garbage/huge timestamp + // cannot overflow i64 during the conversion. + return Some(numeric.saturating_mul(1000)); } None diff --git a/crates/tokscale-core/src/sessions/zed.rs b/crates/tokscale-core/src/sessions/zed.rs index a804b5356..02dc53f78 100644 --- a/crates/tokscale-core/src/sessions/zed.rs +++ b/crates/tokscale-core/src/sessions/zed.rs @@ -180,7 +180,7 @@ fn parse_thread_row(db_path: &Path, row: ZedThreadRow) -> Option let (tokens, message_count) = thread_usage(&thread)?; let timestamp = timestamp_ms(&row, &thread)?; - let mut message = UnifiedMessage::new( + let mut message = UnifiedMessage::new_with_dedup( "zed", model_id, ZED_HOSTED_PROVIDER, @@ -188,9 +188,9 @@ fn parse_thread_row(db_path: &Path, row: ZedThreadRow) -> Option timestamp, tokens, 0.0, + Some(format!("zed:{}", row.id)), ); message.message_count = message_count; - message.dedup_key = Some(format!("zed:{}", row.id)); if let Some(workspace_key) = workspace_key_from_folders( row.folder_paths.as_deref(), From fbd2bd60700d8d615f62490109e387deefb1544f Mon Sep 17 00:00:00 2001 From: Junho Yeo Date: Mon, 22 Jun 2026 17:46:59 +0900 Subject: [PATCH 3/3] style(report): drop useless vec! in test helpers Pre-existing clippy useless-vec warnings in report.rs test helpers, swept to keep the clippy gate clean. --- crates/tokscale-cli/src/commands/report.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tokscale-cli/src/commands/report.rs b/crates/tokscale-cli/src/commands/report.rs index dafecec6a..2d3dd7fe2 100644 --- a/crates/tokscale-cli/src/commands/report.rs +++ b/crates/tokscale-cli/src/commands/report.rs @@ -1328,7 +1328,7 @@ mod tests { #[test] fn cluster_titles_merges_near_duplicates() { - let entries = vec![ + let entries = [ titled_entry("a", "Enhance API Security"), titled_entry("b", "Enhance API security with JWT auth middleware"), titled_entry("c", "Add JWT auth middleware"), @@ -1356,7 +1356,7 @@ mod tests { #[test] fn cluster_titles_keeps_unrelated_apart() { - let entries = vec![ + let entries = [ titled_entry("a", "Add JWT auth middleware"), titled_entry("b", "Update database migration scripts"), titled_entry("c", "Refactor pricing service cache"), @@ -1377,7 +1377,7 @@ mod tests { fn cluster_label_prefers_most_frequent_then_shortest() { // Two identical long titles + one shorter variant: frequency wins, so the // repeated long title is the label even though a shorter one exists. - let entries = vec![ + let entries = [ titled_entry("a", "Add JWT auth middleware"), titled_entry("b", "Add JWT auth middleware"), titled_entry("c", "JWT auth"),