From 170674749163ed8883a00ab78b520af8aed65539 Mon Sep 17 00:00:00 2001 From: Ivan Golovach <20299097+IvGolovach@users.noreply.github.com> Date: Sun, 3 May 2026 14:54:34 -0700 Subject: [PATCH 1/4] fix(gemini): parse current JSONL session usage (#495) Support current Gemini CLI chat recordings by discovering session JSONL files and parsing direct Gemini token events while preserving legacy JSON and headless stats formats. Validation - cargo test -p tokscale-core jsonl - cargo test -p tokscale-core gemini - cargo test -p tokscale-core scan_all_clients_gemini - cargo test -p tokscale-core - cargo clippy -p tokscale-core --all-features -- -D warnings - cargo fmt --all --check - git diff --check - git diff --cached --check --- crates/tokscale-core/src/clients.rs | 2 +- crates/tokscale-core/src/scanner.rs | 34 ++++ crates/tokscale-core/src/sessions/gemini.rs | 182 +++++++++++++++++--- 3 files changed, 194 insertions(+), 24 deletions(-) diff --git a/crates/tokscale-core/src/clients.rs b/crates/tokscale-core/src/clients.rs index a1619b57f..abe4bfafc 100644 --- a/crates/tokscale-core/src/clients.rs +++ b/crates/tokscale-core/src/clients.rs @@ -208,7 +208,7 @@ define_clients!( id: "gemini", root: PathRoot::Home, relative: ".gemini/tmp", - pattern: "*.json", + pattern: "*.json|*.jsonl", headless: false, parse_local: true, submit_default: true diff --git a/crates/tokscale-core/src/scanner.rs b/crates/tokscale-core/src/scanner.rs index f3ae7e761..36f430b95 100644 --- a/crates/tokscale-core/src/scanner.rs +++ b/crates/tokscale-core/src/scanner.rs @@ -188,6 +188,7 @@ pub fn scan_directory(root: &str, pattern: &str) -> Vec { match pattern { "*.json" => file_name.ends_with(".json"), + "*.json|*.jsonl" => file_name.ends_with(".json") || file_name.ends_with(".jsonl"), "*.jsonl" => file_name.ends_with(".jsonl"), // OpenClaw: also match archived transcripts // (.jsonl.deleted., .jsonl.reset.) @@ -1030,6 +1031,26 @@ mod tests { assert!(json_files.iter().all(|p| p.extension().unwrap() == "json")); } + #[test] + fn test_scan_directory_json_or_jsonl_pattern() { + let dir = TempDir::new().unwrap(); + let path = dir.path(); + + File::create(path.join("session.json")).unwrap(); + File::create(path.join("session.jsonl")).unwrap(); + File::create(path.join("session.txt")).unwrap(); + + let session_files = scan_directory(path.to_str().unwrap(), "*.json|*.jsonl"); + assert_eq!(session_files.len(), 2); + assert_eq!( + session_files + .iter() + .map(|path| path.file_name().unwrap().to_str().unwrap()) + .collect::>(), + vec!["session.json", "session.jsonl"] + ); + } + #[test] fn test_scan_directory_jsonl_pattern() { let dir = TempDir::new().unwrap(); @@ -1853,6 +1874,19 @@ mod tests { assert!(result.get(ClientId::OpenCode).is_empty()); } + #[test] + fn test_scan_all_clients_gemini_jsonl_session() { + let dir = TempDir::new().unwrap(); + let home = dir.path(); + let gemini_path = home.join(".gemini/tmp/123/chats"); + fs::create_dir_all(&gemini_path).unwrap(); + File::create(gemini_path.join("session-abc.jsonl")).unwrap(); + + let result = scan_all_clients(home.to_str().unwrap(), &["gemini".to_string()]); + assert_eq!(result.get(ClientId::Gemini).len(), 1); + assert!(result.get(ClientId::Gemini)[0].ends_with("session-abc.jsonl")); + } + #[test] fn test_scan_all_clients_copilot() { let dir = TempDir::new().unwrap(); diff --git a/crates/tokscale-core/src/sessions/gemini.rs b/crates/tokscale-core/src/sessions/gemini.rs index 1fc0ef2f5..7e38f5892 100644 --- a/crates/tokscale-core/src/sessions/gemini.rs +++ b/crates/tokscale-core/src/sessions/gemini.rs @@ -1,7 +1,8 @@ //! Gemini CLI session parser //! -//! Parses JSON session files from ~/.gemini/tmp/* supporting both legacy -//! `session-*.json` files and new UUID-named files in `chats/` directories. +//! Parses JSON and JSONL session files from ~/.gemini/tmp/* supporting legacy +//! `session-*.json` files, UUID-named files in `chats/`, and current +//! `session-*.jsonl` chat recordings. use super::utils::{ extract_i64, extract_string, file_modified_timestamp_ms, parse_timestamp_value, @@ -11,6 +12,7 @@ use super::UnifiedMessage; use crate::TokenBreakdown; use serde::Deserialize; use serde_json::Value; +use std::collections::HashMap; use std::io::{BufRead, BufReader}; use std::path::Path; @@ -146,35 +148,65 @@ fn parse_gemini_session(session: GeminiSession, fallback_timestamp: i64) -> Vec< .and_then(|ts| chrono::DateTime::parse_from_rfc3339(&ts).ok()) .map(|dt| dt.timestamp_millis()) .unwrap_or(fallback_timestamp); - let (input, cache_read) = normalize_gemini_session_input_and_cache( - tokens.input.unwrap_or(0), - tokens.cached.unwrap_or(0), - tokens.output.unwrap_or(0), - tokens.thoughts.unwrap_or(0), - tokens.tool.unwrap_or(0), - tokens.total, - ); - - messages.push(UnifiedMessage::new( - "gemini", + messages.push(build_gemini_token_message( model, - "google", - session_id.clone(), + &session_id, timestamp, - TokenBreakdown { - input, - output: tokens.output.unwrap_or(0).max(0), - cache_read, - cache_write: 0, - reasoning: tokens.thoughts.unwrap_or(0).max(0), - }, - 0.0, + tokens, )); } messages } +fn build_gemini_token_message( + model: String, + session_id: &str, + timestamp: i64, + tokens: GeminiTokens, +) -> UnifiedMessage { + let (input, cache_read) = normalize_gemini_session_input_and_cache( + tokens.input.unwrap_or(0), + tokens.cached.unwrap_or(0), + tokens.output.unwrap_or(0), + tokens.thoughts.unwrap_or(0), + tokens.tool.unwrap_or(0), + tokens.total, + ); + + UnifiedMessage::new( + "gemini", + model, + "google", + session_id.to_string(), + timestamp, + TokenBreakdown { + input, + output: tokens.output.unwrap_or(0).max(0), + cache_read, + cache_write: 0, + reasoning: tokens.thoughts.unwrap_or(0).max(0), + }, + 0.0, + ) +} + +fn parse_direct_gemini_token_message( + value: &Value, + model_hint: Option, + session_id: &str, + fallback_timestamp: i64, +) -> Option { + let model = extract_string(value.get("model")).or(model_hint)?; + let tokens_value = value.get("tokens")?; + let tokens: GeminiTokens = serde_json::from_value(tokens_value.clone()).ok()?; + let timestamp = extract_timestamp_from_value(value).unwrap_or(fallback_timestamp); + + Some(build_gemini_token_message( + model, session_id, timestamp, tokens, + )) +} + fn parse_gemini_headless_jsonl(path: &Path, fallback_timestamp: i64) -> Vec { let file = match std::fs::File::open(path) { Ok(f) => f, @@ -189,6 +221,7 @@ fn parse_gemini_headless_jsonl(path: &Path, fallback_timestamp: i64) -> Vec = None; let reader = BufReader::new(file); let mut messages = Vec::with_capacity(64); + let mut direct_message_indices: HashMap = HashMap::new(); let mut buffer = Vec::with_capacity(4096); for line in reader.lines() { @@ -222,6 +255,36 @@ fn parse_gemini_headless_jsonl(path: &Path, fallback_timestamp: i64) -> Vec Vec { + if value.get("type").and_then(|val| val.as_str()) == Some("gemini") { + if let Some(message) = + parse_direct_gemini_token_message(value, None, session_id, fallback_timestamp) + { + return vec![message]; + } + } + let stats = match value .get("stats") .or_else(|| value.get("result").and_then(|result| result.get("stats"))) @@ -645,6 +716,71 @@ mod tests { assert_eq!(messages[0].tokens.total(), 35); } + #[test] + fn test_parse_gemini_stream_jsonl_direct_tokens() { + let content = r#"{"sessionId":"gemini-session-1","projectHash":"abc123","startTime":"2026-05-01T00:00:00.000Z","lastUpdated":"2026-05-01T00:01:00.000Z"} +{"id":"msg-1","timestamp":"2026-05-01T00:01:00.000Z","type":"gemini","model":"gemini-3.1-pro-preview","tokens":{"input":14918,"output":60,"cached":0,"thoughts":863,"tool":0,"total":15841}}"#; + let dir = TempDir::new().unwrap(); + let chats_dir = dir.path().join(".gemini/tmp/123/chats"); + std::fs::create_dir_all(&chats_dir).unwrap(); + let file_path = chats_dir.join("session-abc.jsonl"); + std::fs::write(&file_path, content).unwrap(); + + let messages = parse_gemini_file(&file_path); + + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].session_id, "gemini-session-1"); + assert_eq!(messages[0].model_id, "gemini-3.1-pro-preview"); + assert_eq!(messages[0].provider_id, "google"); + assert_eq!(messages[0].tokens.input, 14918); + assert_eq!(messages[0].tokens.output, 60); + assert_eq!(messages[0].tokens.cache_read, 0); + assert_eq!(messages[0].tokens.reasoning, 863); + assert_eq!(messages[0].tokens.total(), 15841); + } + + #[test] + fn test_parse_gemini_stream_jsonl_replaces_duplicate_message_id() { + let content = r#"{"type":"gemini","id":"msg-1","model":"gemini-3.1-pro-preview","tokens":{"input":10,"output":1,"cached":0,"thoughts":0,"tool":0,"total":11}} +{"type":"gemini","id":"msg-1","model":"gemini-3.1-pro-preview","tokens":{"input":20,"output":2,"cached":5,"thoughts":3,"tool":0,"total":25}}"#; + let dir = TempDir::new().unwrap(); + let chats_dir = dir.path().join(".gemini/tmp/123/chats"); + std::fs::create_dir_all(&chats_dir).unwrap(); + let file_path = chats_dir.join("session-abc.jsonl"); + std::fs::write(&file_path, content).unwrap(); + + let messages = parse_gemini_file(&file_path); + + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].model_id, "gemini-3.1-pro-preview"); + assert_eq!(messages[0].tokens.input, 15); + assert_eq!(messages[0].tokens.output, 2); + assert_eq!(messages[0].tokens.cache_read, 5); + assert_eq!(messages[0].tokens.reasoning, 3); + assert_eq!(messages[0].tokens.total(), 25); + } + + #[test] + fn test_parse_gemini_json_direct_tokens() { + let json = r#"{"type":"gemini","model":"gemini-3.1-pro-preview","tokens":{"input":20,"output":2,"cached":5,"thoughts":3,"tool":0,"total":25}}"#; + let file = tempfile::Builder::new() + .prefix("session-") + .suffix(".json") + .tempfile() + .unwrap(); + std::fs::write(file.path(), json).unwrap(); + + let messages = parse_gemini_file(file.path()); + + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].model_id, "gemini-3.1-pro-preview"); + assert_eq!(messages[0].tokens.input, 15); + assert_eq!(messages[0].tokens.output, 2); + assert_eq!(messages[0].tokens.cache_read, 5); + assert_eq!(messages[0].tokens.reasoning, 3); + assert_eq!(messages[0].tokens.total(), 25); + } + #[test] fn test_parse_headless_json_clamps_cached_input_overlap() { let json = r#"{"response":"Hi","stats":{"models":{"gemini-2.5-pro":{"tokens":{"prompt":5,"candidates":2,"cached":10}}}}}"#; From e11e0fd7b75a4af303eb87438bca83870554489d Mon Sep 17 00:00:00 2001 From: Ivan Golovach <20299097+IvGolovach@users.noreply.github.com> Date: Sun, 3 May 2026 14:54:51 -0700 Subject: [PATCH 2/4] fix(codex): attribute resumed token counts to later model (#496) Validation - scripts/ledger/check: Not applicable - scripts/ledger/ does not exist in this repository. - git fetch --no-tags origin main:refs/remotes/origin/main: PASS - cargo test -p tokscale-core test_model_only_headless_line_flushes_pending_token_counts: PASS, 1 passed - cargo test -p tokscale-core token_count: PASS, 11 passed - cargo test -p tokscale-core codex_cache: PASS, 2 passed - cargo test -p tokscale-core: PASS, 600 passed, 1 ignored; codebuff 10 passed; hermes 3 passed; doc-tests 0 passed - cargo clippy -p tokscale-core --all-features -- -D warnings: PASS - cargo fmt --all --check: PASS - git diff --check: PASS - git diff --cached --check: PASS - Cyrillic scan for touched source files: PASS, no matches Rollback - git revert HEAD --- crates/tokscale-core/src/lib.rs | 80 ++++++++++ crates/tokscale-core/src/message_cache.rs | 2 +- crates/tokscale-core/src/sessions/codex.rs | 176 +++++++++++++++++++-- 3 files changed, 245 insertions(+), 13 deletions(-) diff --git a/crates/tokscale-core/src/lib.rs b/crates/tokscale-core/src/lib.rs index caa63a372..9a4e4b1e2 100644 --- a/crates/tokscale-core/src/lib.rs +++ b/crates/tokscale-core/src/lib.rs @@ -465,6 +465,13 @@ fn parse_all_messages_with_pricing_with_env_strategy( }; } + if parsed.unresolved_model_events { + return CachedParseOutcome { + messages, + cache_entry: None, + }; + } + let cache_entry = build_codex_cache_entry( path, parsed.messages, @@ -654,6 +661,12 @@ fn parse_all_messages_with_pricing_with_env_strategy( fallback_timestamp, ); + if parsed.unresolved_model_events { + return CachedParseOutcome { + messages, + cache_entry: None, + }; + } let cache_entry = build_codex_cache_entry( path, raw_messages, @@ -3384,6 +3397,73 @@ mod tests { } } + #[test] + #[serial_test::serial] + fn test_codex_cache_does_not_persist_unknown_before_later_turn_context() { + let cache_home = tempfile::TempDir::new().unwrap(); + let source_home = tempfile::TempDir::new().unwrap(); + let original_home = std::env::var("HOME").ok(); + std::env::set_var("HOME", cache_home.path()); + + { + let session_dir = source_home.path().join(".codex/sessions"); + std::fs::create_dir_all(&session_dir).unwrap(); + let path = session_dir.join("session.jsonl"); + std::fs::write( + &path, + concat!( + r#"{"type":"session_meta","payload":{"source":"interactive","model_provider":"openai"}}"#, + "\n", + r#"{"timestamp":"2026-04-27T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3}}}}"#, + "\n" + ), + ) + .unwrap(); + + let initial_messages = parse_all_messages_with_pricing( + source_home.path().to_str().unwrap(), + &["codex".to_string()], + None, + ); + assert_eq!(initial_messages.len(), 1); + assert_eq!(initial_messages[0].model_id, "unknown"); + assert!(message_cache::SourceMessageCache::load() + .get(&path) + .is_none()); + + std::thread::sleep(std::time::Duration::from_millis(5)); + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap(); + file.write_all( + concat!( + r#"{"timestamp":"2026-04-27T10:00:04Z","type":"turn_context","payload":{"model":"gpt-5.5"}}"#, + "\n" + ) + .as_bytes(), + ) + .unwrap(); + file.flush().unwrap(); + + let resumed_messages = parse_all_messages_with_pricing( + source_home.path().to_str().unwrap(), + &["codex".to_string()], + None, + ); + assert_eq!(resumed_messages.len(), 1); + assert_eq!(resumed_messages[0].model_id, "gpt-5.5"); + assert!(message_cache::SourceMessageCache::load() + .get(&path) + .is_some()); + } + + match original_home { + Some(home) => std::env::set_var("HOME", home), + None => std::env::remove_var("HOME"), + } + } + #[test] #[serial_test::serial] fn test_source_cache_does_not_reuse_priced_cost_without_pricing_service() { diff --git a/crates/tokscale-core/src/message_cache.rs b/crates/tokscale-core/src/message_cache.rs index 25dfb1ef1..c17991728 100644 --- a/crates/tokscale-core/src/message_cache.rs +++ b/crates/tokscale-core/src/message_cache.rs @@ -10,7 +10,7 @@ use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::time::UNIX_EPOCH; -const CACHE_SCHEMA_VERSION: u32 = 8; +const CACHE_SCHEMA_VERSION: u32 = 9; const CACHE_FILENAME: &str = "source-message-cache.bin"; const CACHE_LOCK_FILENAME: &str = "source-message-cache.lock"; const MAX_CACHE_FILE_BYTES: u64 = 256 * 1024 * 1024; diff --git a/crates/tokscale-core/src/sessions/codex.rs b/crates/tokscale-core/src/sessions/codex.rs index 1bcecd70c..52dcd5a8f 100644 --- a/crates/tokscale-core/src/sessions/codex.rs +++ b/crates/tokscale-core/src/sessions/codex.rs @@ -164,6 +164,8 @@ pub(crate) struct ParsedCodexFile { pub fallback_timestamp_indices: Vec, pub consumed_offset: u64, pub parse_succeeded: bool, + /// True when model-less token_count rows were emitted without a later model. + pub unresolved_model_events: bool, pub state: CodexParseState, } @@ -187,6 +189,8 @@ fn parse_codex_reader( let mut line = String::with_capacity(4096); let mut consumed_offset = start_offset; let mut parse_succeeded = true; + let mut pending_model_messages = Vec::new(); + let mut unresolved_model_events = false; loop { line.clear(); @@ -231,6 +235,14 @@ fn parse_codex_reader( // Extract model from turn_context if entry.entry_type == "turn_context" { state.current_model = extract_model(&payload); + if let Some(model) = state.current_model.clone() { + flush_pending_model_messages( + &mut pending_model_messages, + &mut messages, + &mut fallback_timestamp_indices, + &model, + ); + } handled = true; } @@ -240,7 +252,13 @@ fn parse_codex_reader( { // Try to extract model from payload if let Some(model) = extract_model(&payload) { - state.current_model = Some(model); + state.current_model = Some(model.clone()); + flush_pending_model_messages( + &mut pending_model_messages, + &mut messages, + &mut fallback_timestamp_indices, + &model, + ); } let info = match payload.info { @@ -250,13 +268,16 @@ fn parse_codex_reader( // Try to extract model from info if let Some(model) = info.model.clone().or(info.model_name.clone()) { - state.current_model = Some(model); + state.current_model = Some(model.clone()); + flush_pending_model_messages( + &mut pending_model_messages, + &mut messages, + &mut fallback_timestamp_indices, + &model, + ); } - let model = state - .current_model - .clone() - .unwrap_or_else(|| "unknown".to_string()); + let model = state.current_model.clone(); // Use last_token_usage as the primary increment source. // Upstream totals are mutable snapshots (compaction, context-window @@ -335,7 +356,7 @@ fn parse_codex_reader( let mut message = UnifiedMessage::new_with_agent( "codex", - model, + model.clone().unwrap_or_else(|| "unknown".to_string()), provider, session_id.to_string(), timestamp, @@ -347,9 +368,13 @@ fn parse_codex_reader( state.session_workspace_key.clone(), state.session_workspace_label.clone(), ); - messages.push(message); - if parsed_timestamp.is_none() { - fallback_timestamp_indices.push(messages.len() - 1); + if model.is_some() { + messages.push(message); + if parsed_timestamp.is_none() { + fallback_timestamp_indices.push(messages.len() - 1); + } + } else { + pending_model_messages.push((message, parsed_timestamp.is_none())); } handled = true; } @@ -365,7 +390,7 @@ fn parse_codex_reader( continue; } - if let Some((mut msg, used_fallback_timestamp)) = parse_codex_headless_line( + let headless_message = parse_codex_headless_line( trimmed, session_id, &mut state.current_model, @@ -373,7 +398,17 @@ fn parse_codex_reader( state.session_provider.as_deref(), &state.session_agent, state.session_is_headless, - ) { + ); + if let Some(model) = state.current_model.clone() { + flush_pending_model_messages( + &mut pending_model_messages, + &mut messages, + &mut fallback_timestamp_indices, + &model, + ); + } + + if let Some((mut msg, used_fallback_timestamp)) = headless_message { msg.set_workspace( state.session_workspace_key.clone(), state.session_workspace_label.clone(), @@ -385,15 +420,41 @@ fn parse_codex_reader( } } + if !pending_model_messages.is_empty() { + unresolved_model_events = true; + flush_pending_model_messages( + &mut pending_model_messages, + &mut messages, + &mut fallback_timestamp_indices, + "unknown", + ); + } + ParsedCodexFile { messages, fallback_timestamp_indices, consumed_offset, parse_succeeded, + unresolved_model_events, state, } } +fn flush_pending_model_messages( + pending_model_messages: &mut Vec<(UnifiedMessage, bool)>, + messages: &mut Vec, + fallback_timestamp_indices: &mut Vec, + model: &str, +) { + for (mut message, used_fallback_timestamp) in pending_model_messages.drain(..) { + message.model_id = model.to_string(); + messages.push(message); + if used_fallback_timestamp { + fallback_timestamp_indices.push(messages.len() - 1); + } + } +} + /// Parse a Codex JSONL file with stateful tracking pub fn parse_codex_file(path: &Path) -> Vec { let file = match std::fs::File::open(path) { @@ -427,6 +488,7 @@ pub(crate) fn parse_codex_file_incremental( fallback_timestamp_indices: Vec::new(), consumed_offset: start_offset, parse_succeeded: false, + unresolved_model_events: false, state, }; } @@ -438,6 +500,7 @@ pub(crate) fn parse_codex_file_incremental( fallback_timestamp_indices: Vec::new(), consumed_offset: start_offset, parse_succeeded: false, + unresolved_model_events: false, state, }; } @@ -738,6 +801,95 @@ mod tests { ); } + #[test] + fn test_token_count_before_turn_context_uses_later_model() { + let file = create_test_file(concat!( + r#"{"type":"session_meta","payload":{"source":"interactive","model_provider":"openai","agent_nickname":"builder","cwd":"/Users/alice/codex-demo"}}"#, + "\n", + r#"{"timestamp":"2026-04-27T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3,"reasoning_output_tokens":1},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3,"reasoning_output_tokens":1}}}}"#, + "\n", + r#"{"timestamp":"2026-04-27T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":15,"cached_input_tokens":3,"output_tokens":5,"reasoning_output_tokens":1},"last_token_usage":{"input_tokens":5,"cached_input_tokens":1,"output_tokens":2,"reasoning_output_tokens":0}}}}"#, + "\n", + r#"{"timestamp":"2026-04-27T10:00:04Z","type":"turn_context","payload":{"model":"gpt-5.5"}}"#, + "\n", + r#"{"timestamp":"2026-04-27T10:00:05Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":22,"cached_input_tokens":4,"output_tokens":7,"reasoning_output_tokens":2},"last_token_usage":{"input_tokens":7,"cached_input_tokens":1,"output_tokens":2,"reasoning_output_tokens":1}}}}"#, + "\n" + )); + + let messages = parse_codex_file(file.path()); + + assert_eq!(messages.len(), 3); + assert_eq!( + messages + .iter() + .map(|message| message.model_id.as_str()) + .collect::>(), + vec!["gpt-5.5", "gpt-5.5", "gpt-5.5"] + ); + assert_eq!( + messages + .iter() + .map(|message| message.workspace_key.as_deref()) + .collect::>(), + vec![ + Some("/Users/alice/codex-demo"), + Some("/Users/alice/codex-demo"), + Some("/Users/alice/codex-demo") + ] + ); + assert_eq!(messages[0].tokens.input, 8); + assert_eq!(messages[0].tokens.output, 3); + assert_eq!(messages[0].tokens.cache_read, 2); + assert_eq!(messages[0].tokens.reasoning, 1); + assert_eq!(messages[1].tokens.input, 4); + assert_eq!(messages[1].tokens.output, 2); + assert_eq!(messages[1].tokens.cache_read, 1); + assert_eq!(messages[1].tokens.reasoning, 0); + assert_eq!(messages[2].tokens.input, 6); + assert_eq!(messages[2].tokens.output, 2); + assert_eq!(messages[2].tokens.cache_read, 1); + assert_eq!(messages[2].tokens.reasoning, 1); + + let parsed = parse_codex_file_incremental(file.path(), 0, CodexParseState::default()); + assert!(!parsed.unresolved_model_events); + } + + #[test] + fn test_token_count_without_model_stays_unknown_but_is_not_cacheable() { + let file = create_test_file(concat!( + r#"{"type":"session_meta","payload":{"source":"interactive","model_provider":"openai"}}"#, + "\n", + r#"{"timestamp":"2026-04-27T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3,"reasoning_output_tokens":1},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3,"reasoning_output_tokens":1}}}}"#, + "\n" + )); + + let parsed = parse_codex_file_incremental(file.path(), 0, CodexParseState::default()); + + assert!(parsed.parse_succeeded); + assert!(parsed.unresolved_model_events); + assert_eq!(parsed.messages.len(), 1); + assert_eq!(parsed.messages[0].model_id, "unknown"); + } + + #[test] + fn test_model_only_headless_line_flushes_pending_token_counts() { + let file = create_test_file(concat!( + r#"{"type":"session_meta","payload":{"source":"interactive","model_provider":"openai"}}"#, + "\n", + r#"{"timestamp":"2026-04-27T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3,"reasoning_output_tokens":1},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":3,"reasoning_output_tokens":1}}}}"#, + "\n", + r#"{"model":"gpt-5.5","type":"metadata"}"#, + "\n" + )); + + let parsed = parse_codex_file_incremental(file.path(), 0, CodexParseState::default()); + + assert!(parsed.parse_succeeded); + assert!(!parsed.unresolved_model_events); + assert_eq!(parsed.messages.len(), 1); + assert_eq!(parsed.messages[0].model_id, "gpt-5.5"); + } + #[test] fn test_parse_reader_marks_failure_on_line_read_error() { let reader = FailAfterFirstLine::new(concat!( From 4a20152f46fe7fc4acfb6d669cc6d669d1bc9822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20K=C4=B1l=C4=B1=C3=A7da=C4=9F=C4=B1?= Date: Mon, 4 May 2026 00:56:06 +0300 Subject: [PATCH 3/4] docs: add deno dx examples (#494) --- README.ja.md | 3 +++ README.ko.md | 3 +++ README.md | 3 +++ README.zh-cn.md | 3 +++ 4 files changed, 12 insertions(+) diff --git a/README.ja.md b/README.ja.md index 712b825e0..955fe51ad 100644 --- a/README.ja.md +++ b/README.ja.md @@ -161,6 +161,9 @@ npx tokscale@latest # またはbunxを使用 bunx tokscale@latest +# たはdeno dxを使用 +dx tokscale@latest + # ライトモード（テーブルレンダリングのみ） npx tokscale@latest --light ``` diff --git a/README.ko.md b/README.ko.md index c8e1d494b..d208de570 100644 --- a/README.ko.md +++ b/README.ko.md @@ -161,6 +161,9 @@ npx tokscale@latest # 또는 bunx 사용 bunx tokscale@latest +# 또는 deno dx 사용 +dx tokscale@latest + # 라이트 모드 (테이블 렌더링만) npx tokscale@latest --light ``` diff --git a/README.md b/README.md index 08ee5bb1e..1818be665 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,9 @@ npx tokscale@latest # Or use bunx bunx tokscale@latest +# Or use deno dx +dx tokscale@latest + # Light mode (table rendering only) npx tokscale@latest --light ``` diff --git a/README.zh-cn.md b/README.zh-cn.md index 359ea0cdc..6abcdeec1 100644 --- a/README.zh-cn.md +++ b/README.zh-cn.md @@ -161,6 +161,9 @@ npx tokscale@latest # 或使用 bunx bunx tokscale@latest +# 或使用 deno dx +dx tokscale@latest + # 轻量模式（仅表格渲染） npx tokscale@latest --light ``` From dcf8c25d21d1b993f08bbc49aa0d5360ad8f7d60 Mon Sep 17 00:00:00 2001 From: shidevil Date: Mon, 4 May 2026 05:13:51 +0000 Subject: [PATCH 4/4] fix(tui): 'r' refreshes all tabs including usage Co-Authored-By: Claude Opus 4.7 --- crates/tokscale-cli/src/tui/app.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tokscale-cli/src/tui/app.rs b/crates/tokscale-cli/src/tui/app.rs index 39de21e6c..21a0f4ff8 100644 --- a/crates/tokscale-cli/src/tui/app.rs +++ b/crates/tokscale-cli/src/tui/app.rs @@ -477,9 +477,7 @@ impl App { self.set_status("Refresh already in progress"); } else { self.needs_reload = true; - if self.current_tab == Tab::Usage { - self.fetch_subscription_usage(); - } + self.fetch_subscription_usage(); } } KeyCode::Char('R') if key.modifiers.contains(KeyModifiers::SHIFT) => {