diff --git a/crates/ironclaw_filesystem/src/catalog.rs b/crates/ironclaw_filesystem/src/catalog.rs index 850fa4a19ad..32dc59fd30d 100644 --- a/crates/ironclaw_filesystem/src/catalog.rs +++ b/crates/ironclaw_filesystem/src/catalog.rs @@ -305,6 +305,24 @@ impl RootFilesystem for CompositeRootFilesystem { .await } + async fn head_seq( + &self, + path: &VirtualPath, + from: SeqNo, + ) -> Result, FilesystemError> { + self.matching_mount(path)? + .backend + .head_seq(path, from) + .await + } + + async fn reserve_sequence(&self, path: &VirtualPath) -> Result { + self.matching_mount(path)? + .backend + .reserve_sequence(path) + .await + } + // ── Legacy bytes plane ── async fn read_file(&self, path: &VirtualPath) -> Result, FilesystemError> { diff --git a/crates/ironclaw_filesystem/src/in_memory.rs b/crates/ironclaw_filesystem/src/in_memory.rs index 72659cc1bcf..d90e10d7d9d 100644 --- a/crates/ironclaw_filesystem/src/in_memory.rs +++ b/crates/ironclaw_filesystem/src/in_memory.rs @@ -50,6 +50,7 @@ struct State { entries: HashMap, indexes: HashMap>, event_logs: HashMap>, + sequences: HashMap, } /// In-memory backend serving the full unified [`RootFilesystem`] surface. @@ -64,6 +65,7 @@ impl InMemoryBackend { entries: HashMap::new(), indexes: HashMap::new(), event_logs: HashMap::new(), + sequences: HashMap::new(), }), } } @@ -145,6 +147,8 @@ impl RootFilesystem for InMemoryBackend { state .entries .retain(|key, _| !key.as_str().starts_with(&prefix)); + clear_event_logs_under(&mut state.event_logs, path.as_str(), &prefix); + clear_sequences_under(&mut state.sequences, path.as_str(), &prefix); return Ok(()); } let prefix = with_trailing_slash(path.as_str()); @@ -152,6 +156,8 @@ impl RootFilesystem for InMemoryBackend { state .entries .retain(|key, _| !key.as_str().starts_with(&prefix)); + clear_event_logs_under(&mut state.event_logs, path.as_str(), &prefix); + clear_sequences_under(&mut state.sequences, path.as_str(), &prefix); if state.entries.len() == before { return Err(FilesystemError::NotFound { path: path.clone(), @@ -432,6 +438,17 @@ impl RootFilesystem for InMemoryBackend { .collect()) } + async fn reserve_sequence(&self, path: &VirtualPath) -> Result { + let mut state = self.state.lock().await; + let next = state + .sequences + .entry(path.as_str().to_string()) + .or_insert_with(|| SeqNo::ZERO.next()); + let reserved = *next; + *next = next.next(); + Ok(reserved) + } + // Legacy bytes ops — default impls in the trait route them through put/get // and use our native implementations. The only one needing an explicit // impl is the required-method `list_dir`, which we already overrode above. @@ -588,6 +605,28 @@ fn with_trailing_slash(s: &str) -> String { } } +/// Drop any reserved sequence counter for `exact` and every path under +/// `prefix` (the trailing-slash form of `exact`). Mirrors the entries-delete +/// subtree semantics so a delete/recreate of the same path restarts its +/// sequence from 1 instead of resuming stale state. The exact match is kept +/// separate from the prefix scan so a sibling sharing a string prefix +/// (`/a/b` vs `/a/bc`) is not swept. +fn clear_sequences_under(sequences: &mut HashMap, exact: &str, prefix: &str) { + sequences.retain(|key, _| key.as_str() != exact && !key.starts_with(prefix)); +} + +/// Drop the append-event log for `exact` and every log under `prefix`. Append-only +/// finalized assistant messages live in `event_logs`, so a delete/recreate of the +/// same thread path would otherwise rehydrate stale append-log history. Mirrors +/// the entries/sequences subtree semantics. +fn clear_event_logs_under( + event_logs: &mut HashMap>, + exact: &str, + prefix: &str, +) { + event_logs.retain(|key, _| key.as_str() != exact && !key.starts_with(prefix)); +} + fn first_segment(s: &str) -> (&str, bool) { match s.find('/') { Some(idx) => (&s[..idx], true), @@ -667,6 +706,46 @@ mod tests { assert!(fs.append_batch(&log, Vec::new()).await.unwrap().is_empty()); } + #[tokio::test] + async fn reserve_sequence_assigns_path_local_monotonic_values() { + let fs = InMemoryBackend::new(); + let thread_a = vpath("/threads/a/message_sequence"); + let thread_b = vpath("/threads/b/message_sequence"); + + assert_eq!(fs.reserve_sequence(&thread_a).await.unwrap().get(), 1); + assert_eq!(fs.reserve_sequence(&thread_a).await.unwrap().get(), 2); + assert_eq!(fs.reserve_sequence(&thread_b).await.unwrap().get(), 1); + assert_eq!(fs.reserve_sequence(&thread_a).await.unwrap().get(), 3); + } + + #[tokio::test] + async fn deleting_a_subtree_clears_its_reserved_sequences() { + // Counters are now path-scoped in a side table rather than embedded in + // the thread record, so delete must sweep them or a delete/recreate of + // the same path would resume from stale state. A sibling that merely + // shares a string prefix must NOT be swept. + let fs = InMemoryBackend::new(); + let seq_path = vpath("/threads/a/message_sequence"); + let sibling = vpath("/threads/ab/message_sequence"); + let thread_dir = vpath("/threads/a"); + + assert_eq!(fs.reserve_sequence(&seq_path).await.unwrap().get(), 1); + assert_eq!(fs.reserve_sequence(&seq_path).await.unwrap().get(), 2); + assert_eq!(fs.reserve_sequence(&sibling).await.unwrap().get(), 1); + + // Materialize an entry so the directory delete has something to remove, + // then delete the whole /threads/a subtree. + fs.put(&seq_path, Entry::bytes(vec![1]), CasExpectation::Any) + .await + .unwrap(); + fs.delete(&thread_dir).await.unwrap(); + + // The deleted subtree's counter restarts from 1. + assert_eq!(fs.reserve_sequence(&seq_path).await.unwrap().get(), 1); + // The string-prefix sibling /threads/ab is untouched. + assert_eq!(fs.reserve_sequence(&sibling).await.unwrap().get(), 2); + } + #[tokio::test] async fn cas_absent_rejects_when_present() { let fs = InMemoryBackend::new(); @@ -899,6 +978,48 @@ mod tests { assert!(matches!(err, FilesystemError::NotFound { .. })); } + #[tokio::test] + async fn delete_sweeps_append_event_logs_under_path() { + // Append-only finalized assistant messages live in the per-thread + // append log. Deleting a thread must clear that log so a later + // recreate of the same path does not replay stale history. This covers + // both delete branches: an exact append-log path, and a thread-root + // directory delete whose log lives under the subtree. + let fs = InMemoryBackend::new(); + + // Exact-entry branch: an entry and an append log share the deleted + // path. Deleting the entry must also sweep the co-located log. + let exact = vpath("/threads/exact"); + fs.put(&exact, Entry::bytes(vec![1]), CasExpectation::Absent) + .await + .unwrap(); + fs.append(&exact, b"finalized-a".to_vec()).await.unwrap(); + assert_eq!(fs.tail(&exact, SeqNo::ZERO).await.unwrap().len(), 1); + fs.delete(&exact).await.unwrap(); + assert!(fs.tail(&exact, SeqNo::ZERO).await.unwrap().is_empty()); + + // Directory branch: the thread root has no exact entry, but a child + // entry and an append log live under it. Deleting the root must sweep + // the log too. + let thread_root = vpath("/threads/t1"); + let thread_doc = vpath("/threads/t1/thread.json"); + let message_log = vpath("/threads/t1/messages_append.log"); + fs.put(&thread_doc, Entry::bytes(vec![1]), CasExpectation::Absent) + .await + .unwrap(); + fs.append(&message_log, b"finalized-b".to_vec()) + .await + .unwrap(); + assert_eq!(fs.tail(&message_log, SeqNo::ZERO).await.unwrap().len(), 1); + + fs.delete(&thread_root).await.unwrap(); + + // History is gone, and recreating the same thread starts from an empty + // log rather than resurrecting the finalized message. + assert!(fs.get(&thread_doc).await.unwrap().is_none()); + assert!(fs.tail(&message_log, SeqNo::ZERO).await.unwrap().is_empty()); + } + #[tokio::test] async fn list_dir_upgrades_to_directory_on_later_child_discovery() { // PR #3659 reviewer fix: with `or_insert`, the first discovery diff --git a/crates/ironclaw_filesystem/src/libsql.rs b/crates/ironclaw_filesystem/src/libsql.rs index b92cc4058b1..9e5b046a0e8 100644 --- a/crates/ironclaw_filesystem/src/libsql.rs +++ b/crates/ironclaw_filesystem/src/libsql.rs @@ -960,6 +960,25 @@ impl RootFilesystem for LibSqlRootFilesystem { if deleted == 0 { return Err(not_found(path.clone(), FilesystemOperation::Delete)); } + // Sweep the append-event log for this path and its subtree. Append-only + // finalized assistant messages live in `root_filesystem_events`, so a + // delete/recreate of the same thread would otherwise replay stale + // history from the old log. Mirrors the entries-delete predicate above. + conn.execute( + "DELETE FROM root_filesystem_events WHERE path = ?1 OR path LIKE ?2 ESCAPE '!'", + libsql::params![path.as_str(), child_path_like_pattern(path)], + ) + .await + .map_err(|error| libsql_db_error(path.clone(), FilesystemOperation::Delete, error))?; + // Sweep any reserved sequence counter for this path and its subtree so + // a delete/recreate restarts sequences from 1 rather than resuming + // stale state. Mirrors the entries-delete predicate above. + conn.execute( + "DELETE FROM root_filesystem_sequences WHERE path = ?1 OR path LIKE ?2 ESCAPE '!'", + libsql::params![path.as_str(), child_path_like_pattern(path)], + ) + .await + .map_err(|error| libsql_db_error(path.clone(), FilesystemOperation::Delete, error))?; Ok(()) } @@ -1157,6 +1176,39 @@ impl RootFilesystem for LibSqlRootFilesystem { } } + async fn reserve_sequence(&self, path: &VirtualPath) -> Result { + let conn = self.connect().await?; + let mut rows = conn + .query( + r#" + INSERT INTO root_filesystem_sequences (path, next_seq, updated_at) + VALUES (?1, 2, strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ON CONFLICT(path) DO UPDATE SET + next_seq = root_filesystem_sequences.next_seq + 1, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + RETURNING next_seq - 1 + "#, + libsql::params![path.as_str()], + ) + .await + .map_err(|error| { + libsql_db_error(path.clone(), FilesystemOperation::ReserveSeq, error) + })?; + let row = rows + .next() + .await + .map_err(|error| libsql_db_error(path.clone(), FilesystemOperation::ReserveSeq, error))? + .ok_or_else(|| FilesystemError::Backend { + path: path.clone(), + operation: FilesystemOperation::ReserveSeq, + reason: "sequence reservation returned no row".to_string(), + })?; + let seq_raw: i64 = row.get(0).map_err(|error| { + libsql_db_error(path.clone(), FilesystemOperation::ReserveSeq, error) + })?; + seq_no_from_i64(path, seq_raw, FilesystemOperation::ReserveSeq) + } + async fn create_dir_all(&self, path: &VirtualPath) -> Result<(), FilesystemError> { let conn = self.connect().await?; let transaction = conn.transaction().await.map_err(|error| { @@ -1218,6 +1270,7 @@ async fn run_libsql_migrations_inner(conn: &libsql::Connection) -> Result<(), Fi ensure_libsql_records_columns(conn).await?; ensure_libsql_index_specs_table(conn).await?; ensure_libsql_events_table(conn).await?; + ensure_libsql_sequences_table(conn).await?; Ok(()) } @@ -1634,6 +1687,14 @@ async fn ensure_libsql_events_table(conn: &libsql::Connection) -> Result<(), Fil Ok(()) } +#[cfg(feature = "libsql")] +async fn ensure_libsql_sequences_table(conn: &libsql::Connection) -> Result<(), FilesystemError> { + conn.execute_batch(LIBSQL_SEQUENCES_SCHEMA) + .await + .map_err(|error| infrastructure_libsql_error(FilesystemOperation::ReserveSeq, error))?; + Ok(()) +} + #[cfg(feature = "libsql")] fn seq_no_from_i64( path: &VirtualPath, @@ -1931,6 +1992,15 @@ CREATE INDEX IF NOT EXISTS idx_root_filesystem_events_path_seq ON root_filesystem_events(path, seq); "#; +#[cfg(feature = "libsql")] +const LIBSQL_SEQUENCES_SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS root_filesystem_sequences ( + path TEXT PRIMARY KEY, + next_seq INTEGER NOT NULL CHECK (next_seq > 0), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); +"#; + #[cfg(test)] mod tests { //! Deterministic regression tests for libSQL behaviours that aren't diff --git a/crates/ironclaw_filesystem/src/postgres.rs b/crates/ironclaw_filesystem/src/postgres.rs index deaadfe39a1..bcda9ab95e2 100644 --- a/crates/ironclaw_filesystem/src/postgres.rs +++ b/crates/ironclaw_filesystem/src/postgres.rs @@ -748,6 +748,26 @@ impl RootFilesystem for PostgresRootFilesystem { } } + async fn reserve_sequence(&self, path: &VirtualPath) -> Result { + let client = self.client().await?; + let row = cached_query_one( + &client, + r#" + INSERT INTO root_filesystem_sequences (path, next_seq, updated_at) + VALUES ($1, 2, NOW()) + ON CONFLICT (path) DO UPDATE SET + next_seq = root_filesystem_sequences.next_seq + 1, + updated_at = NOW() + RETURNING next_seq - 1 AS reserved + "#, + &[&path.as_str()], + ) + .await + .map_err(|error| db_error(path.clone(), FilesystemOperation::ReserveSeq, error))?; + let reserved: i64 = row.get("reserved"); + seq_no_from_i64(path, reserved, FilesystemOperation::ReserveSeq) + } + async fn create_dir_all(&self, path: &VirtualPath) -> Result<(), FilesystemError> { let mut client = self.client().await?; let transaction = client @@ -1395,6 +1415,27 @@ async fn postgres_delete_with_client( if deleted == 0 { return Err(not_found(path.clone(), FilesystemOperation::Delete)); } + // Sweep the append-event log for this path and its subtree. Append-only + // finalized assistant messages live in `root_filesystem_events`, so a + // delete/recreate of the same thread would otherwise replay stale history + // from the old log. Mirrors the entries-delete predicate above. + cached_execute( + client, + "DELETE FROM root_filesystem_events WHERE path = $1 OR (path >= $2 AND path < $3)", + &[&path.as_str(), &prefix_lower, &prefix_upper], + ) + .await + .map_err(|error| db_error(path.clone(), FilesystemOperation::Delete, error))?; + // Sweep any reserved sequence counter for this path and its subtree so a + // delete/recreate restarts sequences from 1 rather than resuming stale + // state. Mirrors the entries-delete predicate above. + cached_execute( + client, + "DELETE FROM root_filesystem_sequences WHERE path = $1 OR (path >= $2 AND path < $3)", + &[&path.as_str(), &prefix_lower, &prefix_upper], + ) + .await + .map_err(|error| db_error(path.clone(), FilesystemOperation::Delete, error))?; Ok(()) } @@ -1712,6 +1753,8 @@ const POSTGRES_ROOT_FILESYSTEM_SCHEMA: &str = concat!( include_str!("../../../migrations/V30__root_filesystem_events.sql"), "\n", include_str!("../../../migrations/V31__root_filesystem_path_collation.sql"), + "\n", + include_str!("../../../migrations/V32__root_filesystem_sequences.sql"), ); #[cfg(all(test, feature = "postgres"))] diff --git a/crates/ironclaw_filesystem/src/root.rs b/crates/ironclaw_filesystem/src/root.rs index 15aa204a258..04d1fa9ea26 100644 --- a/crates/ironclaw_filesystem/src/root.rs +++ b/crates/ironclaw_filesystem/src/root.rs @@ -260,6 +260,17 @@ pub trait RootFilesystem: Send + Sync { Ok(records.into_iter().map(|record| record.seq).max()) } + /// Reserve and return the next monotonic sequence number for `path`. + /// + /// Unlike [`append`](Self::append), this sequence is scoped to `path` + /// rather than the backend's global event table. Consumers use it to + /// assign row-native ordering keys without rewriting a shared metadata + /// record under CAS. A failed follow-up write may leave a gap; callers must + /// rely on monotonicity, not contiguity. + async fn reserve_sequence(&self, path: &VirtualPath) -> Result { + unsupported(path, FilesystemOperation::ReserveSeq) + } + // ─── Legacy bytes plane (DEPRECATED — removed after consumer migration) ─ // // The methods below predate the unified [`put`]/[`get`] surface and exist diff --git a/crates/ironclaw_filesystem/src/scoped.rs b/crates/ironclaw_filesystem/src/scoped.rs index b109aaec6bf..8f03b3a1f4a 100644 --- a/crates/ironclaw_filesystem/src/scoped.rs +++ b/crates/ironclaw_filesystem/src/scoped.rs @@ -237,6 +237,17 @@ where self.root.head_seq(&virtual_path, from).await } + /// Reserve a path-local monotonic sequence number. + pub async fn reserve_sequence( + &self, + scope: &ResourceScope, + path: &ScopedPath, + ) -> Result { + let virtual_path = + self.resolve_with_permission(scope, path, FilesystemOperation::ReserveSeq)?; + self.root.reserve_sequence(&virtual_path).await + } + // ─── Legacy bytes-plane methods (DEPRECATED — transitional) ─────────── /// **DEPRECATED — use [`read_bytes`](Self::read_bytes) or @@ -437,7 +448,8 @@ fn operation_allowed(permissions: &MountPermissions, operation: FilesystemOperat | FilesystemOperation::CreateDirAll | FilesystemOperation::EnsureIndex | FilesystemOperation::BeginTxn - | FilesystemOperation::Append => permissions.write, + | FilesystemOperation::Append + | FilesystemOperation::ReserveSeq => permissions.write, FilesystemOperation::ListDir => permissions.list, FilesystemOperation::Stat => permissions.read || permissions.list, FilesystemOperation::Delete => permissions.delete, diff --git a/crates/ironclaw_filesystem/src/types.rs b/crates/ironclaw_filesystem/src/types.rs index ab77032d219..bdfa078186e 100644 --- a/crates/ironclaw_filesystem/src/types.rs +++ b/crates/ironclaw_filesystem/src/types.rs @@ -13,7 +13,8 @@ use crate::record::RecordVersion; /// *intent* of an operation against the underlying [`MountPermissions`] /// surface and are reused by the unified `put`/`get` ops as their permission /// witness — `put` is a write, `get` is a read. The newer variants -/// (`Query`, `EnsureIndex`, `BeginTxn`, `Append`, `Tail`, `HeadSeq`) describe +/// (`Query`, `EnsureIndex`, `BeginTxn`, `Append`, `Tail`, `HeadSeq`, +/// `ReserveSeq`) describe /// operations that have no analogue in the legacy enum. /// /// `AppendFile` is the legacy byte-plane append onto a regular file; `Append` @@ -39,6 +40,10 @@ pub enum FilesystemOperation { /// replay/live boundary. Distinct from `Tail` (which streams records) so a /// head_seq failure surfaces under its own operation in logs/errors. HeadSeq, + /// Path-local monotonic sequence reservation. Used by row-shaped stores + /// that need an atomic per-record-set counter without rewriting a shared + /// JSON snapshot just to assign the next sequence. + ReserveSeq, } impl std::fmt::Display for FilesystemOperation { @@ -59,6 +64,7 @@ impl std::fmt::Display for FilesystemOperation { Self::Append => "append", Self::Tail => "tail", Self::HeadSeq => "head_seq", + Self::ReserveSeq => "reserve_seq", }) } } diff --git a/crates/ironclaw_first_party_extensions/src/coding/paths.rs b/crates/ironclaw_first_party_extensions/src/coding/paths.rs index 1910f412a07..086bb77277a 100644 --- a/crates/ironclaw_first_party_extensions/src/coding/paths.rs +++ b/crates/ironclaw_first_party_extensions/src/coding/paths.rs @@ -127,7 +127,8 @@ pub(super) fn operation_allowed( FilesystemOperation::Query => permissions.read && permissions.list, FilesystemOperation::EnsureIndex | FilesystemOperation::BeginTxn - | FilesystemOperation::Append => permissions.write, + | FilesystemOperation::Append + | FilesystemOperation::ReserveSeq => permissions.write, FilesystemOperation::Tail | FilesystemOperation::HeadSeq => permissions.read, } } diff --git a/crates/ironclaw_host_runtime/src/invocation_services.rs b/crates/ironclaw_host_runtime/src/invocation_services.rs index 0c796e277e2..8711be65412 100644 --- a/crates/ironclaw_host_runtime/src/invocation_services.rs +++ b/crates/ironclaw_host_runtime/src/invocation_services.rs @@ -348,7 +348,8 @@ fn operation_allowed(permissions: &MountPermissions, operation: FilesystemOperat | FilesystemOperation::CreateDirAll | FilesystemOperation::EnsureIndex | FilesystemOperation::BeginTxn - | FilesystemOperation::Append => permissions.write, + | FilesystemOperation::Append + | FilesystemOperation::ReserveSeq => permissions.write, FilesystemOperation::ListDir => permissions.list, FilesystemOperation::Stat => permissions.read || permissions.list, FilesystemOperation::Delete => permissions.delete, diff --git a/crates/ironclaw_threads/src/contract.rs b/crates/ironclaw_threads/src/contract.rs index aad89ab695a..361e4dcee37 100644 --- a/crates/ironclaw_threads/src/contract.rs +++ b/crates/ironclaw_threads/src/contract.rs @@ -353,6 +353,14 @@ pub struct AppendAssistantDraftRequest { pub content: MessageContent, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppendFinalizedAssistantMessageRequest { + pub scope: ThreadScope, + pub thread_id: ThreadId, + pub turn_run_id: String, + pub content: MessageContent, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct AppendToolResultReferenceRequest { pub scope: ThreadScope, diff --git a/crates/ironclaw_threads/src/filesystem_service.rs b/crates/ironclaw_threads/src/filesystem_service.rs index 368f182647f..f1d59112c60 100644 --- a/crates/ironclaw_threads/src/filesystem_service.rs +++ b/crates/ironclaw_threads/src/filesystem_service.rs @@ -45,7 +45,7 @@ use chrono::Utc; use futures::{StreamExt, future::join_all}; use ironclaw_filesystem::{ CasExpectation, ContentType, Entry, FileType, FilesystemError, FilesystemOperation, Filter, - Page, RecordVersion, RootFilesystem, ScopedFilesystem, + Page, RecordVersion, RootFilesystem, ScopedFilesystem, SeqNo, }; use ironclaw_host_api::{HostApiError, InvocationId, ResourceScope, ScopedPath, ThreadId}; use serde::{Deserialize, Serialize}; @@ -58,16 +58,16 @@ use crate::title::derive_title_from_message; use crate::{ AcceptInboundMessageRequest, AcceptedInboundMessage, AcceptedInboundMessageReplay, AppendAssistantDraftRequest, AppendCapabilityDisplayPreviewRequest, - AppendToolResultReferenceRequest, CapabilityDisplayPreviewEnvelope, ContextMessage, - ContextMessages, ContextWindow, CreateSummaryArtifactRequest, EnsureThreadRequest, - LatestThreadMessageRequest, ListThreadsForScopeRequest, ListThreadsForScopeResponse, - LoadContextMessagesRequest, LoadContextWindowRequest, MessageContent, MessageKind, - MessageStatus, ProviderToolCallReferenceEnvelope, RedactMessageRequest, - ReplayAcceptedInboundMessageRequest, SessionThreadError, SessionThreadRecord, - SessionThreadService, SummaryArtifact, SummaryModelContextPolicy, ThreadHistory, - ThreadHistoryRequest, ThreadMessageId, ThreadMessageRange, ThreadMessageRangeRequest, - ThreadMessageRecord, ThreadScope, ToolResultReferenceEnvelope, UpdateAssistantDraftRequest, - UpdateToolResultReferenceRequest, + AppendFinalizedAssistantMessageRequest, AppendToolResultReferenceRequest, + CapabilityDisplayPreviewEnvelope, ContextMessage, ContextMessages, ContextWindow, + CreateSummaryArtifactRequest, EnsureThreadRequest, LatestThreadMessageRequest, + ListThreadsForScopeRequest, ListThreadsForScopeResponse, LoadContextMessagesRequest, + LoadContextWindowRequest, MessageContent, MessageKind, MessageStatus, + ProviderToolCallReferenceEnvelope, RedactMessageRequest, ReplayAcceptedInboundMessageRequest, + SessionThreadError, SessionThreadRecord, SessionThreadService, SummaryArtifact, + SummaryModelContextPolicy, ThreadHistory, ThreadHistoryRequest, ThreadMessageId, + ThreadMessageRange, ThreadMessageRangeRequest, ThreadMessageRecord, ThreadScope, + ToolResultReferenceEnvelope, UpdateAssistantDraftRequest, UpdateToolResultReferenceRequest, }; use message_lookup_index::MessageLookupIndexStore; use message_sequence_index::{MessageSequenceIndexStore, message_sequence_index_entry_for_message}; @@ -223,7 +223,12 @@ where .get(&scope.to_resource_scope(), &path) .await? else { - return Ok(None); + let Some(events) = self.read_message_append_events(scope, thread_id).await? else { + return Ok(None); + }; + return Ok(events + .into_iter() + .find(|(record, _)| record.message_id == message_id)); }; let record = deserialize::(&versioned.entry.body)?; if &record.thread_id != thread_id || record.message_id != message_id { @@ -320,6 +325,101 @@ where } } + async fn append_message_event( + &self, + scope: &ThreadScope, + thread_id: &ThreadId, + message: &ThreadMessageRecord, + ) -> Result { + let path = message_append_log_path(scope, thread_id)?; + let payload = serialize_pretty(&StoredThreadMessageRecord::from(message))?; + match self + .filesystem + .append(&scope.to_resource_scope(), &path, payload) + .await + { + Ok(_) => Ok(true), + Err(FilesystemError::Unsupported { + operation: FilesystemOperation::Append, + .. + }) => Ok(false), + Err(error) => Err(error.into()), + } + } + + async fn read_message_append_events( + &self, + scope: &ThreadScope, + thread_id: &ThreadId, + ) -> Result>, SessionThreadError> { + let path = message_append_log_path(scope, thread_id)?; + let events = match self + .filesystem + .tail(&scope.to_resource_scope(), &path, SeqNo::ZERO) + .await + { + Ok(events) => events, + Err(FilesystemError::Unsupported { + operation: FilesystemOperation::Tail, + .. + }) => return Ok(None), + Err(FilesystemError::NotFound { .. }) => return Ok(Some(Vec::new())), + Err(error) => return Err(error.into()), + }; + + let mut messages = Vec::with_capacity(events.len()); + for event in events { + let message = deserialize::(&event.payload)?; + if &message.thread_id == thread_id { + messages.push((message, RecordVersion::from_backend(event.seq.get()))); + } + } + messages.sort_by_key(|(message, _)| message.sequence); + Ok(Some(messages)) + } + + async fn list_message_append_events( + &self, + scope: &ThreadScope, + thread_id: &ThreadId, + ) -> Result, SessionThreadError> { + Ok(self + .read_message_append_events(scope, thread_id) + .await? + .unwrap_or_default() + .into_iter() + .map(|(message, _)| message) + .collect()) + } + + async fn merge_message_append_events( + &self, + scope: &ThreadScope, + thread_id: &ThreadId, + messages: &mut Vec, + ) -> Result<(), SessionThreadError> { + let mut event_messages = self.list_message_append_events(scope, thread_id).await?; + if event_messages.is_empty() { + return Ok(()); + } + // File-authoritative merge: a per-message file always wins over its + // append-log entry. The log only contributes messages that have no + // file yet (a finalized message written solely via + // `append_message_event`). This matches the single-message read in + // `read_message_versioned` (file first, log fallback) and lets + // `apply_message_update` materialize a file on mutation + // (redaction/status change) and have that file shadow the original + // log entry. Were the log to win, a redacted message's file would be + // masked by its stale log record. + event_messages.retain(|event| { + !messages + .iter() + .any(|existing| existing.message_id == event.message_id) + }); + messages.append(&mut event_messages); + Ok(()) + } + async fn try_write_new_message_transactionally( &self, scope: &ThreadScope, @@ -375,33 +475,17 @@ where thread_id: thread_id.clone(), }); }; - let mut stored = deserialize::(&versioned_thread.entry.body)?; + let stored = deserialize::(&versioned_thread.entry.body)?; if &stored.record.scope != scope || &stored.record.thread_id != thread_id { txn.rollback().await; return Err(SessionThreadError::UnknownThread { thread_id: thread_id.clone(), }); } - let assigned = stored.next_sequence; - stored.next_sequence = assigned + 1; - stored.record.updated_at = Some(Utc::now()); - let thread_entry = Self::thread_entry(&stored)?; - if let Err(error) = txn - .put( - &thread_virtual_path, - thread_entry, - CasExpectation::Version(versioned_thread.version), - ) - .await - { - txn.rollback().await; - match error { - FilesystemError::VersionMismatch { .. } => continue, - error => return Err(error.into()), - }; - } - message.sequence = assigned; + if message.sequence == 0 { + message.sequence = self.reserve_sequence(scope, thread_id).await?; + } let message_entry = Self::message_entry(message)?; if let Err(error) = txn .put(&message_virtual_path, message_entry, CasExpectation::Absent) @@ -430,8 +514,16 @@ where )); } - txn.commit().await?; - return Ok(TransactionalMessageWrite::Written); + match txn.commit().await { + Ok(()) => return Ok(TransactionalMessageWrite::Written), + // Optimistic-concurrency conflict on the thread record: another + // writer committed between our `get` and `commit`. Retry the + // whole transaction — this is the bounded CAS-retry budget the + // loop exists to provide. (libSQL/in-memory never reach here; + // they return `Unsupported` from `begin` above.) + Err(FilesystemError::VersionMismatch { .. }) => continue, + Err(error) => return Err(error.into()), + } } Err(SessionThreadError::Backend(format!( @@ -489,6 +581,8 @@ where offset = offset.saturating_add(entry_count as u64); } + self.merge_message_append_events(scope, thread_id, &mut messages) + .await?; messages.sort_by_key(|message| message.sequence); Ok(messages) } @@ -526,6 +620,8 @@ where messages.push(record); } } + self.merge_message_append_events(scope, thread_id, &mut messages) + .await?; messages.sort_by_key(|message| message.sequence); Ok(messages) } @@ -671,32 +767,13 @@ where &self, scope: &ThreadScope, thread_id: &ThreadId, - next_sequence: u64, + _next_sequence: u64, ) -> Result, SessionThreadError> { - let index_store = MessageSequenceIndexStore::new(self.filesystem.as_ref()); - for sequence in 1..next_sequence { - let Some(index) = index_store.read(scope, thread_id, sequence).await? else { - return Ok(self - .list_thread_messages(scope, thread_id) - .await? - .into_iter() - .find(|message| message.kind == MessageKind::User)); - }; - let Some((message, _)) = self - .read_message_versioned(scope, thread_id, index.message_id) - .await? - else { - return Ok(self - .list_thread_messages(scope, thread_id) - .await? - .into_iter() - .find(|message| message.kind == MessageKind::User)); - }; - if message.kind == MessageKind::User { - return Ok(Some(message)); - } - } - Ok(None) + Ok(self + .list_thread_messages(scope, thread_id) + .await? + .into_iter() + .find(|message| message.kind == MessageKind::User)) } async fn materialize_message_range( @@ -714,7 +791,6 @@ where thread_id: thread_id.clone(), })? .0; - let through_sequence = through_sequence.min(thread.next_sequence.saturating_sub(1)); let messages = match self .list_thread_messages_range_indexed(scope, thread_id, after_sequence, through_sequence) .await? @@ -929,14 +1005,62 @@ where Ok(()) } - /// Read-modify-write the `next_sequence` counter on the thread record - /// with optimistic CAS and bounded retry. Returns the sequence - /// assigned to the caller (i.e. `next_sequence` before the bump) plus - /// a clone of the post-bump record. + /// Reserve a per-thread message sequence without rewriting the thread + /// metadata record. SQL-backed filesystems serve this with an atomic + /// path-local counter row; older/backends without the sequence primitive + /// fall back to the legacy `thread.json` CAS counter. async fn reserve_sequence( &self, scope: &ThreadScope, thread_id: &ThreadId, + ) -> Result { + let (stored, _) = self + .read_thread_versioned(scope, thread_id) + .await? + .ok_or_else(|| SessionThreadError::UnknownThread { + thread_id: thread_id.clone(), + })?; + // Migration safety: a thread that already assigned message sequences + // under the legacy per-thread-record counter (`next_sequence > 1`) must + // keep using it. The native path-local counter starts at 1 for a path + // with no row, so switching an *existing* thread onto it would restart + // at 1 and collide with messages already at sequences 1..N — corrupting + // ordering and clobbering the sequence index on instances that predate + // this change. New/empty threads (`next_sequence == 1`, no messages + // yet) take the fast native counter; because the native path never + // rewrites `next_sequence`, such a thread's record stays at 1 and + // deterministically keeps using the native path for its whole life, + // while a pre-existing thread stays on the legacy counter for its whole + // life. No thread ever switches counters mid-stream. + if stored.next_sequence > 1 { + return self + .reserve_sequence_via_thread_record(scope, thread_id) + .await; + } + let sequence_path = message_sequence_counter_path(scope, thread_id)?; + match self + .filesystem + .reserve_sequence(&scope.to_resource_scope(), &sequence_path) + .await + { + Ok(sequence) => return Ok(sequence.get()), + Err(FilesystemError::Unsupported { + operation: FilesystemOperation::ReserveSeq, + .. + }) => {} + Err(error) => return Err(error.into()), + } + self.reserve_sequence_via_thread_record(scope, thread_id) + .await + } + + /// Legacy fallback for backends that cannot atomically reserve a + /// path-local sequence. This preserves compatibility but retains the old + /// shared-thread-record CAS bottleneck. + async fn reserve_sequence_via_thread_record( + &self, + scope: &ThreadScope, + thread_id: &ThreadId, ) -> Result { let path = thread_record_path(scope, thread_id)?; for _ in 0..FILESYSTEM_CAS_RETRIES { @@ -974,6 +1098,77 @@ where ))) } + /// Stamp `thread.updated_at = now` at a turn boundary (inbound accept, + /// finalized assistant append) so `list_threads_for_scope` orders by + /// genuine recency without scanning transcripts. This is the single + /// extra thread-record write per turn that the native (non-thread- + /// record) `reserve_sequence` path otherwise avoids — bounded to turn + /// boundaries, not per message or per token. + /// + /// Best-effort under contention: a `VersionMismatch` means a concurrent + /// activity write already advanced `updated_at` (the safe direction), so + /// after the bounded retry budget we return `Ok(())` rather than failing + /// the enclosing append — the message itself is already durably written. + /// A non-CAS backend error still propagates. + async fn touch_thread_updated_at( + &self, + scope: &ThreadScope, + thread_id: &ThreadId, + ) -> Result<(), SessionThreadError> { + let path = thread_record_path(scope, thread_id)?; + for _ in 0..FILESYSTEM_CAS_RETRIES { + // The `?` propagates real read errors; the `else` handles only the + // genuine "thread deleted between append and stamp" case — there is + // nothing to order in the sidebar, so nothing to touch. + let Some((mut stored, version)) = self.read_thread_versioned(scope, thread_id).await? + else { + return Ok(()); + }; + stored.record.updated_at = Some(Utc::now()); + let entry = Self::thread_entry(&stored)?; + match put_with_cas( + self.filesystem.as_ref(), + &scope.to_resource_scope(), + &path, + entry, + CasExpectation::Version(version), + ) + .await + { + Ok(()) => return Ok(()), + Err(PutError::VersionMismatch) => continue, + Err(PutError::Other(error)) => return Err(error), + } + } + // Retry budget exhausted purely on `VersionMismatch`: concurrent + // writers are advancing `updated_at` themselves, so the thread is + // already surfacing as recently-active. Don't fail the append over a + // best-effort ordering stamp. + tracing::debug!( + thread_id = %thread_id.as_str(), + "thread updated_at touch lost CAS race within retry budget; ordering stamp left to the concurrent writer", + ); + Ok(()) + } + + /// Best-effort recency stamp for after-commit call sites. The message is + /// already durably written when these run, so a touch failure must not + /// fail the enclosing operation: `accept_inbound_message` permits requests + /// without an idempotency key, and propagating an error here could make an + /// un-idempotent caller retry and duplicate the message. Logs and + /// continues; the advisory `updated_at` stamp simply stays at its prior + /// value until the next activity. + async fn touch_thread_updated_at_best_effort(&self, scope: &ThreadScope, thread_id: &ThreadId) { + if let Err(error) = self.touch_thread_updated_at(scope, thread_id).await { + // silent-ok: recency stamp is advisory after the message is durable. + tracing::debug!( + ?error, + thread_id = %thread_id.as_str(), + "message persisted but thread recency touch failed", + ); + } + } + /// Read-modify-write a single message record with optimistic CAS and /// bounded retry. The `mutate` closure projects the staged record onto /// its new shape. @@ -989,10 +1184,42 @@ where { let path = message_record_path(scope, thread_id, message_id)?; for _ in 0..FILESYSTEM_CAS_RETRIES { - let (mut message, version) = self - .read_message_versioned(scope, thread_id, message_id) + // Read the individual message file first. A finalized message that + // was written solely to the per-thread append log (see + // `append_finalized_assistant_message`) has no file yet; in that + // case materialize the file on first mutation with + // `CasExpectation::Absent`. Because `merge_message_append_events` + // is file-authoritative, the materialized (e.g. redacted) file + // then shadows the original append-log entry on reads. A + // concurrent materialization races to `Absent` and loses with + // `VersionMismatch`, so the retry re-reads and takes the + // file-versioned path. + let (mut message, cas) = match self + .filesystem + .get(&scope.to_resource_scope(), &path) .await? - .ok_or(SessionThreadError::UnknownMessage { message_id })?; + { + Some(versioned) => { + let record = deserialize::(&versioned.entry.body)?; + if &record.thread_id != thread_id || record.message_id != message_id { + return Err(SessionThreadError::UnknownMessage { message_id }); + } + (record, CasExpectation::Version(versioned.version)) + } + None => { + let Some(events) = self.read_message_append_events(scope, thread_id).await? + else { + return Err(SessionThreadError::UnknownMessage { message_id }); + }; + let Some((record, _)) = events + .into_iter() + .find(|(record, _)| record.message_id == message_id) + else { + return Err(SessionThreadError::UnknownMessage { message_id }); + }; + (record, CasExpectation::Absent) + } + }; mutate(&mut message)?; let entry = Self::message_entry(&message)?; match put_with_cas( @@ -1000,7 +1227,7 @@ where &scope.to_resource_scope(), &path, entry, - CasExpectation::Version(version), + cas, ) .await { @@ -1261,6 +1488,13 @@ where } }; + // Inbound user message is thread activity — stamp recency so the + // sidebar surfaces this thread first. Best-effort: the message is + // already durable, so a touch failure must not fail (and retry) the + // accept. + self.touch_thread_updated_at_best_effort(&scope, &thread_id) + .await; + Ok(AcceptedInboundMessage { thread_id, message_id, @@ -1406,6 +1640,109 @@ where Ok(message) } + async fn append_finalized_assistant_message( + &self, + request: AppendFinalizedAssistantMessageRequest, + ) -> Result { + if let Some(existing) = self + .find_assistant_message_by_run( + &request.scope, + &request.thread_id, + &request.turn_run_id, + None, + ) + .await? + { + if existing.status != MessageStatus::Draft { + // Idempotent-retry repair. A prior call may have appended the + // finalized message event (durable) but failed before writing + // the sequence index. On retry we resolve the finalized message + // through the append-log fallback and would otherwise return + // without the index, leaving a durable LLM message invisible to + // indexed range/context reads. Re-assert the index before + // returning — `write_new` is idempotent when the same message + // is already indexed, so this is a no-op on the fully-persisted + // path and a repair on the partial-failure path. Fail loud (`?`) + // rather than silently returning an unindexed message. + self.write_message_sequence_index(&request.scope, &request.thread_id, &existing) + .await?; + return Ok(existing); + } + let content = request.content.clone(); + let finalized = self + .apply_message_update( + &request.scope, + &request.thread_id, + existing.message_id, + |message| { + ensure_draft(message)?; + message.status = MessageStatus::Finalized; + message.content = Some(content.clone().into_text()); + message.attachments = Vec::new(); + Ok(()) + }, + ) + .await?; + // Finalizing the in-flight draft is thread activity — stamp recency + // (best-effort; the draft update above is already durable). + self.touch_thread_updated_at_best_effort(&request.scope, &request.thread_id) + .await; + return Ok(finalized); + } + let sequence = self + .reserve_sequence(&request.scope, &request.thread_id) + .await?; + let message = ThreadMessageRecord { + message_id: ThreadMessageId::new(), + thread_id: request.thread_id.clone(), + sequence, + kind: MessageKind::Assistant, + status: MessageStatus::Finalized, + actor_id: None, + source_binding_id: None, + reply_target_binding_id: None, + turn_id: None, + turn_run_id: Some(request.turn_run_id), + tool_result_ref: None, + tool_result_provider_call: None, + content: Some(request.content.into_text()), + attachments: Vec::new(), + redaction_ref: None, + }; + if self + .append_message_event(&request.scope, &request.thread_id, &message) + .await? + { + // Append-only path: the log event is durable, but unlike + // `write_new_message` it wrote neither the per-message file nor + // the sequence index. Write the sequence index so indexed range + // reads (`list_thread_messages_range` and the summary/context + // paths built on it) surface this finalized message — full-history + // and context reads already see it via `merge_message_append_events`, + // but the index-backed range path would otherwise omit it. The id + // resolves through `read_message_versioned`'s append-log fallback + // until a later mutation materializes the file. Treated as + // must-write (matching `write_new_message`), not best-effort, + // because a missing index entry silently drops the message from + // range reads. + self.write_message_sequence_index(&request.scope, &request.thread_id, &message) + .await?; + } else { + self.write_new_message( + &request.scope, + &request.thread_id, + &message, + "finalized assistant message", + ) + .await?; + } + // Finalized assistant reply is thread activity — stamp recency + // (best-effort; the append above is already durable). + self.touch_thread_updated_at_best_effort(&request.scope, &request.thread_id) + .await; + Ok(message) + } + async fn append_tool_result_reference( &self, request: AppendToolResultReferenceRequest, @@ -1685,14 +2022,22 @@ where .ok_or_else(|| SessionThreadError::UnknownThread { thread_id: thread_id.clone(), })?; - self.apply_message_update(scope, thread_id, message_id, |message| { - ensure_draft(message)?; - message.status = MessageStatus::Finalized; - message.content = Some(content.clone().into_text()); - message.attachments = Vec::new(); - Ok(()) - }) - .await + let finalized = self + .apply_message_update(scope, thread_id, message_id, |message| { + ensure_draft(message)?; + message.status = MessageStatus::Finalized; + message.content = Some(content.clone().into_text()); + message.attachments = Vec::new(); + Ok(()) + }) + .await?; + // Finalizing the assistant draft is thread activity — stamp recency + // (best-effort; the finalize above is already durable). Without this, + // the draft/update/finalize path would leave active threads stale in + // the `updated_at`-sorted sidebar. + self.touch_thread_updated_at_best_effort(scope, thread_id) + .await; + Ok(finalized) } async fn redact_message( @@ -1967,22 +2312,21 @@ where &self, request: ListThreadsForScopeRequest, ) -> Result { - // Per-request work scales with total thread count, not page - // size. Activity ordering (newest interaction first) requires - // every record's timestamp, so we read all records under the - // scope, sort by activity, then slice the requested page. The - // current `ScopedFilesystem` port exposes neither a + // Per-request work scales with thread *count*, not transcript + // *volume*. Activity ordering (newest interaction first) needs each + // thread's `updated_at`, so we read all records under the scope + // (one `get` per thread, concurrency-bounded), sort by that + // timestamp, then slice the requested page. Activity ordering itself + // never scans transcripts — `updated_at` is stamped at turn + // boundaries by `touch_thread_updated_at`, so it already reflects + // recency. The heavier title-derivation probes still read transcripts, + // but only for the sliced page. The current `ScopedFilesystem` port + // exposes neither a // cursor-paginated directory listing nor a timestamp index, and - // adding either belongs upstream of this crate. Acceptable today - // because: - // * local-dev / single-tenant deployments keep the per-scope - // thread count bounded (per agent + project + owner). - // * the record-read fan-out is concurrency-bounded, and the - // heavier title-derivation probes still run only for the - // sliced page. - // When a tenant grows past low thousands of threads under a - // single scope, replace this with a storage-level paginator - // (e.g. a secondary index keyed by `(scope, updated_at)`). + // adding either belongs upstream of this crate. When a tenant grows + // past low thousands of threads under a single scope, replace this + // with a storage-level paginator (e.g. a secondary index keyed by + // `(scope, updated_at)`). let limit = request .limit .map(|n| (n as usize).clamp(1, LIST_THREADS_MAX_PAGE_SIZE)) @@ -2050,11 +2394,15 @@ where } } } - // Newest activity first (`updated_at`, falling back to - // `created_at`). Legacy records without timestamps sort last. - // Tie-break on thread_id ascending so the order is stable and - // opaque cursors stay resumable — and to match the web sidebar's - // `byActivityDesc` comparator. + // Newest activity first, by `thread.updated_at` (falling back to + // `created_at`). The append paths stamp `updated_at` at turn + // boundaries via `touch_thread_updated_at` (inbound accept + + // finalized assistant append), so this is a true cross-thread + // recency signal — and it costs only the per-thread record read + // already done above, with no per-thread transcript scan. Tie-break + // on thread_id ascending so the order is stable and opaque cursors + // stay resumable — and to match the web sidebar's `byActivityDesc` + // comparator. listed.sort_by(|(a, _), (b, _)| { let a_key = a.updated_at.or(a.created_at); let b_key = b.updated_at.or(b.created_at); @@ -2239,6 +2587,26 @@ fn message_record_path( )) } +fn message_sequence_counter_path( + scope: &ThreadScope, + thread_id: &ThreadId, +) -> Result { + scoped_path(&format!( + "{}/message_sequence", + thread_root_string(scope, thread_id) + )) +} + +fn message_append_log_path( + scope: &ThreadScope, + thread_id: &ThreadId, +) -> Result { + scoped_path(&format!( + "{}/message_appends", + thread_root_string(scope, thread_id) + )) +} + fn summaries_root( scope: &ThreadScope, thread_id: &ThreadId, @@ -2760,4 +3128,95 @@ mod tests { assert!(record_key.starts_with("sha256-")); assert_eq!(record_key.len(), "sha256-".len() + 64); } + + /// Migration safety: a thread that already assigned message sequences under + /// the legacy per-thread-record counter (`next_sequence > 1`) must keep + /// resuming from it, never restart at 1 on the native path-local counter — + /// otherwise deploying this change onto an existing instance would collide + /// new messages with the existing 1..N sequences. New/empty threads + /// (`next_sequence == 1`) take the native counter. + #[tokio::test] + async fn reserve_sequence_resumes_existing_thread_counter_not_native_restart() { + use ironclaw_filesystem::{CasExpectation, InMemoryBackend, ScopedFilesystem}; + use ironclaw_host_api::{ + MountAlias, MountGrant, MountPermissions, MountView, ThreadId, VirtualPath, + }; + + use super::{FilesystemSessionThreadService, thread_record_path}; + use crate::{EnsureThreadRequest, SessionThreadService}; + + let backend = std::sync::Arc::new(InMemoryBackend::new()); + let mounts = MountView::new(vec![MountGrant::new( + MountAlias::new("/threads").unwrap(), + VirtualPath::new("/tenants/t/users/u/threads").unwrap(), + MountPermissions::read_write_list_delete(), + )]) + .unwrap(); + let scoped = std::sync::Arc::new(ScopedFilesystem::with_fixed_view(backend, mounts)); + let service = FilesystemSessionThreadService::new(scoped); + let scope = ThreadScope { + tenant_id: TenantId::new("t").unwrap(), + agent_id: AgentId::new("a").unwrap(), + project_id: Some(ProjectId::new("p").unwrap()), + owner_user_id: Some(UserId::new("u").unwrap()), + mission_id: None, + }; + + // Fresh thread (next_sequence == 1) → native path-local counter from 1. + let fresh = ThreadId::new("fresh").unwrap(); + service + .ensure_thread(EnsureThreadRequest { + scope: scope.clone(), + thread_id: Some(fresh.clone()), + created_by_actor_id: "actor".into(), + title: None, + metadata_json: None, + }) + .await + .unwrap(); + assert_eq!(service.reserve_sequence(&scope, &fresh).await.unwrap(), 1); + assert_eq!(service.reserve_sequence(&scope, &fresh).await.unwrap(), 2); + + // Simulate a pre-existing thread: bump its on-disk `next_sequence` to 5 + // (as the legacy per-record counter would have, for a thread with + // messages at sequences 1..4) while leaving the native counter absent. + let existing = ThreadId::new("existing").unwrap(); + service + .ensure_thread(EnsureThreadRequest { + scope: scope.clone(), + thread_id: Some(existing.clone()), + created_by_actor_id: "actor".into(), + title: None, + metadata_json: None, + }) + .await + .unwrap(); + let (mut stored, version) = service + .read_thread_versioned(&scope, &existing) + .await + .unwrap() + .unwrap(); + stored.next_sequence = 5; + let record_path = thread_record_path(&scope, &existing).unwrap(); + service + .filesystem + .put( + &scope.to_resource_scope(), + &record_path, + FilesystemSessionThreadService::::thread_entry(&stored).unwrap(), + CasExpectation::Version(version), + ) + .await + .unwrap(); + + // Reservation resumes the legacy counter at 5, not the native restart 1. + assert_eq!( + service.reserve_sequence(&scope, &existing).await.unwrap(), + 5 + ); + assert_eq!( + service.reserve_sequence(&scope, &existing).await.unwrap(), + 6 + ); + } } diff --git a/crates/ironclaw_threads/src/in_memory.rs b/crates/ironclaw_threads/src/in_memory.rs index ef742e62ed2..161cb32f5ec 100644 --- a/crates/ironclaw_threads/src/in_memory.rs +++ b/crates/ironclaw_threads/src/in_memory.rs @@ -15,15 +15,16 @@ use crate::title::derive_thread_title; use crate::{ AcceptInboundMessageRequest, AcceptedInboundMessage, AcceptedInboundMessageReplay, AppendAssistantDraftRequest, AppendCapabilityDisplayPreviewRequest, - AppendToolResultReferenceRequest, CapabilityDisplayPreviewEnvelope, ContextMessage, - ContextMessages, ContextWindow, CreateSummaryArtifactRequest, EnsureThreadRequest, - LatestThreadMessageRequest, ListThreadsForScopeRequest, ListThreadsForScopeResponse, - LoadContextMessagesRequest, LoadContextWindowRequest, MessageContent, MessageKind, - MessageStatus, RedactMessageRequest, ReplayAcceptedInboundMessageRequest, SessionThreadError, - SessionThreadRecord, SessionThreadService, SummaryArtifact, SummaryModelContextPolicy, - ThreadHistory, ThreadHistoryRequest, ThreadMessageId, ThreadMessageRange, - ThreadMessageRangeRequest, ThreadMessageRecord, ThreadScope, ToolResultReferenceEnvelope, - UpdateAssistantDraftRequest, UpdateToolResultReferenceRequest, + AppendFinalizedAssistantMessageRequest, AppendToolResultReferenceRequest, + CapabilityDisplayPreviewEnvelope, ContextMessage, ContextMessages, ContextWindow, + CreateSummaryArtifactRequest, EnsureThreadRequest, LatestThreadMessageRequest, + ListThreadsForScopeRequest, ListThreadsForScopeResponse, LoadContextMessagesRequest, + LoadContextWindowRequest, MessageContent, MessageKind, MessageStatus, RedactMessageRequest, + ReplayAcceptedInboundMessageRequest, SessionThreadError, SessionThreadRecord, + SessionThreadService, SummaryArtifact, SummaryModelContextPolicy, ThreadHistory, + ThreadHistoryRequest, ThreadMessageId, ThreadMessageRange, ThreadMessageRangeRequest, + ThreadMessageRecord, ThreadScope, ToolResultReferenceEnvelope, UpdateAssistantDraftRequest, + UpdateToolResultReferenceRequest, }; #[derive(Debug, Clone, Default)] @@ -301,6 +302,48 @@ impl SessionThreadService for InMemorySessionThreadService { Ok(message) } + async fn append_finalized_assistant_message( + &self, + request: AppendFinalizedAssistantMessageRequest, + ) -> Result { + let mut state = self.state.lock().await; + let thread = get_thread_mut(&mut state, &request.scope, &request.thread_id)?; + if let Some(existing) = thread.messages.iter_mut().find(|message| { + message.kind == MessageKind::Assistant + && message.turn_run_id.as_deref() == Some(request.turn_run_id.as_str()) + }) { + if existing.status == MessageStatus::Draft { + existing.status = MessageStatus::Finalized; + existing.content = Some(request.content.into_text()); + existing.attachments = Vec::new(); + } + return Ok(existing.clone()); + } + let message = ThreadMessageRecord { + message_id: ThreadMessageId::new(), + thread_id: request.thread_id.clone(), + sequence: thread.next_sequence, + kind: MessageKind::Assistant, + status: MessageStatus::Finalized, + actor_id: None, + source_binding_id: None, + reply_target_binding_id: None, + turn_id: None, + turn_run_id: Some(request.turn_run_id), + tool_result_ref: None, + tool_result_provider_call: None, + content: Some(request.content.into_text()), + attachments: Vec::new(), + redaction_ref: None, + }; + thread.next_sequence += 1; + // Appending a message is thread activity; bump the last-activity + // stamp so activity-ordered listings surface this thread first. + thread.record.updated_at = Some(Utc::now()); + thread.messages.push(message.clone()); + Ok(message) + } + async fn append_tool_result_reference( &self, request: AppendToolResultReferenceRequest, diff --git a/crates/ironclaw_threads/src/lib.rs b/crates/ironclaw_threads/src/lib.rs index 07111c8a390..1e379a37116 100644 --- a/crates/ironclaw_threads/src/lib.rs +++ b/crates/ironclaw_threads/src/lib.rs @@ -35,16 +35,16 @@ pub use capability_display_preview::{ pub use contract::{ AcceptInboundMessageRequest, AcceptedInboundMessage, AcceptedInboundMessageReplay, AppendAssistantDraftRequest, AppendCapabilityDisplayPreviewRequest, - AppendToolResultReferenceRequest, ContextImageAttachment, ContextMessage, ContextMessages, - ContextWindow, CreateSummaryArtifactRequest, EnsureThreadRequest, - FinalizedAssistantMessageByRunRequest, GOAL_STATEMENT_MAX_CHARS, GoalStatement, - LatestThreadMessageRequest, ListThreadsForScopeRequest, ListThreadsForScopeResponse, - LoadContextMessagesRequest, LoadContextWindowRequest, MessageContent, MessageKind, - MessageStatus, RedactMessageRequest, ReplayAcceptedInboundMessageRequest, SessionThreadRecord, - SummaryArtifact, SummaryKind, SummaryModelContextPolicy, ThreadGoal, ThreadHistory, - ThreadHistoryRequest, ThreadMessageRange, ThreadMessageRangeRequest, ThreadMessageRecord, - ThreadScope, UpdateAssistantDraftRequest, UpdateThreadGoalRequest, - UpdateToolResultReferenceRequest, + AppendFinalizedAssistantMessageRequest, AppendToolResultReferenceRequest, + ContextImageAttachment, ContextMessage, ContextMessages, ContextWindow, + CreateSummaryArtifactRequest, EnsureThreadRequest, FinalizedAssistantMessageByRunRequest, + GOAL_STATEMENT_MAX_CHARS, GoalStatement, LatestThreadMessageRequest, + ListThreadsForScopeRequest, ListThreadsForScopeResponse, LoadContextMessagesRequest, + LoadContextWindowRequest, MessageContent, MessageKind, MessageStatus, RedactMessageRequest, + ReplayAcceptedInboundMessageRequest, SessionThreadRecord, SummaryArtifact, SummaryKind, + SummaryModelContextPolicy, ThreadGoal, ThreadHistory, ThreadHistoryRequest, ThreadMessageRange, + ThreadMessageRangeRequest, ThreadMessageRecord, ThreadScope, UpdateAssistantDraftRequest, + UpdateThreadGoalRequest, UpdateToolResultReferenceRequest, }; pub use error::SessionThreadError; pub use identifiers::{SummaryArtifactId, ThreadMessageId}; diff --git a/crates/ironclaw_threads/src/service.rs b/crates/ironclaw_threads/src/service.rs index 80619ef6b7c..ae9be8bd801 100644 --- a/crates/ironclaw_threads/src/service.rs +++ b/crates/ironclaw_threads/src/service.rs @@ -6,14 +6,15 @@ use ironclaw_host_api::ThreadId; use crate::{ AcceptInboundMessageRequest, AcceptedInboundMessage, AcceptedInboundMessageReplay, AppendAssistantDraftRequest, AppendCapabilityDisplayPreviewRequest, - AppendToolResultReferenceRequest, ContextMessages, ContextWindow, CreateSummaryArtifactRequest, - EnsureThreadRequest, FinalizedAssistantMessageByRunRequest, LatestThreadMessageRequest, - ListThreadsForScopeRequest, ListThreadsForScopeResponse, LoadContextMessagesRequest, - LoadContextWindowRequest, MessageContent, RedactMessageRequest, - ReplayAcceptedInboundMessageRequest, SessionThreadError, SessionThreadRecord, SummaryArtifact, - ThreadGoal, ThreadHistory, ThreadHistoryRequest, ThreadMessageId, ThreadMessageRange, - ThreadMessageRangeRequest, ThreadMessageRecord, ThreadScope, UpdateAssistantDraftRequest, - UpdateThreadGoalRequest, UpdateToolResultReferenceRequest, + AppendFinalizedAssistantMessageRequest, AppendToolResultReferenceRequest, ContextMessages, + ContextWindow, CreateSummaryArtifactRequest, EnsureThreadRequest, + FinalizedAssistantMessageByRunRequest, LatestThreadMessageRequest, ListThreadsForScopeRequest, + ListThreadsForScopeResponse, LoadContextMessagesRequest, LoadContextWindowRequest, + MessageContent, RedactMessageRequest, ReplayAcceptedInboundMessageRequest, SessionThreadError, + SessionThreadRecord, SummaryArtifact, ThreadGoal, ThreadHistory, ThreadHistoryRequest, + ThreadMessageId, ThreadMessageRange, ThreadMessageRangeRequest, ThreadMessageRecord, + ThreadScope, UpdateAssistantDraftRequest, UpdateThreadGoalRequest, + UpdateToolResultReferenceRequest, }; /// Canonical Reborn session thread and transcript boundary. @@ -55,6 +56,28 @@ pub trait SessionThreadService: Send + Sync { request: AppendAssistantDraftRequest, ) -> Result; + async fn append_finalized_assistant_message( + &self, + request: AppendFinalizedAssistantMessageRequest, + ) -> Result { + let scope = request.scope; + let thread_id = request.thread_id; + let content = request.content; + let message = self + .append_assistant_draft(AppendAssistantDraftRequest { + scope: scope.clone(), + thread_id: thread_id.clone(), + turn_run_id: request.turn_run_id, + content: content.clone(), + }) + .await?; + if message.status != crate::MessageStatus::Draft { + return Ok(message); + } + self.finalize_assistant_message(&scope, &thread_id, message.message_id, content) + .await + } + async fn append_tool_result_reference( &self, request: AppendToolResultReferenceRequest, @@ -321,6 +344,15 @@ where self.as_ref().append_assistant_draft(request).await } + async fn append_finalized_assistant_message( + &self, + request: AppendFinalizedAssistantMessageRequest, + ) -> Result { + self.as_ref() + .append_finalized_assistant_message(request) + .await + } + async fn append_tool_result_reference( &self, request: AppendToolResultReferenceRequest, diff --git a/crates/ironclaw_threads/tests/filesystem_message_range_contract.rs b/crates/ironclaw_threads/tests/filesystem_message_range_contract.rs index ffcb6e0c578..d448dc5cdae 100644 --- a/crates/ironclaw_threads/tests/filesystem_message_range_contract.rs +++ b/crates/ironclaw_threads/tests/filesystem_message_range_contract.rs @@ -10,10 +10,10 @@ use ironclaw_host_api::{ ThreadId, UserId, VirtualPath, }; use ironclaw_threads::{ - AcceptInboundMessageRequest, CreateSummaryArtifactRequest, EnsureThreadRequest, - FilesystemSessionThreadService, MessageContent, SessionThreadError, SessionThreadService, - SummaryKind, SummaryModelContextPolicy, ThreadMessageId, ThreadMessageRangeRequest, - ThreadScope, + AcceptInboundMessageRequest, AppendFinalizedAssistantMessageRequest, + CreateSummaryArtifactRequest, EnsureThreadRequest, FilesystemSessionThreadService, + MessageContent, SessionThreadError, SessionThreadService, SummaryKind, + SummaryModelContextPolicy, ThreadMessageId, ThreadMessageRangeRequest, ThreadScope, }; #[tokio::test] @@ -43,6 +43,112 @@ async fn filesystem_store_range_read_returns_only_requested_sequences() { ); } +/// A finalized assistant message stored only via the append log (no +/// per-message file) must still be written into the sequence index, otherwise +/// indexed range reads — which back `list_thread_messages_range`, summaries, +/// and compaction — would silently omit it from threads that also have +/// indexed messages. +#[tokio::test] +async fn filesystem_store_range_read_includes_append_only_finalized_message() { + let fixture = RangeFixture::new("fs-range-append", "tenant-range-append").await; + // Two indexed user messages (sequences 1, 2) so the index is non-empty — + // `list_thread_messages_range_indexed` will not fall back to a full scan. + fixture.seed_messages("event", 2).await; + + let finalized = fixture + .service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: fixture.scope.clone(), + thread_id: fixture.thread_id.clone(), + turn_run_id: "run-append-only".into(), + content: MessageContent::text("assistant reply"), + }) + .await + .unwrap(); + assert_eq!(finalized.sequence, 3); + + // The append-only branch must have actually run: the finalized message has + // no per-message file (it lives solely in the append log). Without this + // guard the test would still pass if `append_message_event` returned false + // and `write_new_message` created the normal per-message file instead. + assert!( + !fixture.message_file_exists(&finalized.message_id).await, + "finalized assistant message must be append-only (no per-message file)" + ); + + // The append-only finalize path must have written the sequence index entry. + assert_eq!( + fixture.index_entry_names().await, + vec![ + "00000000000000000001.json", + "00000000000000000002.json", + "00000000000000000003.json", + ] + ); + + // The indexed range read includes the append-only finalized message (its id + // resolves through `read_message_versioned`'s append-log fallback). + assert_eq!(fixture.range_sequences(0, 3).await, vec![1, 2, 3]); + assert_eq!( + fixture.range_contents(2, 3).await, + vec!["assistant reply".to_string()] + ); +} + +/// If a finalized assistant message was appended to the log but the process +/// died before its sequence index was written, an idempotent retry (same +/// `turn_run_id`) must repair the missing index rather than returning the +/// already-finalized message with no indexed entry — otherwise a durable LLM +/// message stays invisible to indexed range reads. +#[tokio::test] +async fn filesystem_append_finalized_assistant_message_retry_repairs_missing_sequence_index() { + let fixture = RangeFixture::new("fs-range-repair", "tenant-range-repair").await; + + let first = fixture + .service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: fixture.scope.clone(), + thread_id: fixture.thread_id.clone(), + turn_run_id: "run-repair".into(), + content: MessageContent::text("assistant reply"), + }) + .await + .unwrap(); + assert_eq!(first.sequence, 1); + assert_eq!( + fixture.index_entry_names().await, + vec!["00000000000000000001.json"] + ); + + // Simulate the partial-persistence failure: the append-log event survived, + // but the sequence index entry is gone. + fixture.delete_sequence_index(1).await; + assert!(fixture.index_entry_names().await.is_empty()); + + // Idempotent retry with the same turn_run_id resolves the finalized + // message via the append-log fallback and must repair the index. + let retried = fixture + .service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: fixture.scope.clone(), + thread_id: fixture.thread_id.clone(), + turn_run_id: "run-repair".into(), + content: MessageContent::text("assistant reply"), + }) + .await + .unwrap(); + assert_eq!(retried.message_id, first.message_id); + assert_eq!(retried.sequence, 1); + assert_eq!( + fixture.index_entry_names().await, + vec!["00000000000000000001.json"], + "idempotent retry must repair the missing sequence index" + ); + + // The repaired index makes the message visible to indexed range reads. + assert_eq!(fixture.range_sequences(0, 1).await, vec![1]); +} + #[tokio::test] async fn filesystem_store_range_read_falls_back_when_sequence_index_has_gap() { let fixture = RangeFixture::new("fs-range-gap", "tenant-range-gap").await; @@ -194,6 +300,17 @@ impl RangeFixture { .unwrap(); } + async fn message_file_exists(&self, message_id: &ThreadMessageId) -> bool { + self.scoped + .get( + &self.scope.to_resource_scope(), + &self.message_path(&message_id.to_string()), + ) + .await + .unwrap() + .is_some() + } + async fn delete_message(&self, message_id: ThreadMessageId) { self.scoped .delete( diff --git a/crates/ironclaw_threads/tests/filesystem_session_thread_contract.rs b/crates/ironclaw_threads/tests/filesystem_session_thread_contract.rs index 92dbc7bb851..3c9c749832a 100644 --- a/crates/ironclaw_threads/tests/filesystem_session_thread_contract.rs +++ b/crates/ironclaw_threads/tests/filesystem_session_thread_contract.rs @@ -28,8 +28,9 @@ use ironclaw_host_api::{ }; use ironclaw_threads::{ AcceptInboundMessageRequest, AppendAssistantDraftRequest, - AppendCapabilityDisplayPreviewRequest, AppendToolResultReferenceRequest, AttachmentKind, - AttachmentRef, CapabilityDisplayPreviewEnvelope, CapabilityDisplayPreviewEnvelopeInput, + AppendCapabilityDisplayPreviewRequest, AppendFinalizedAssistantMessageRequest, + AppendToolResultReferenceRequest, AttachmentKind, AttachmentRef, + CapabilityDisplayPreviewEnvelope, CapabilityDisplayPreviewEnvelopeInput, CapabilityDisplayPreviewStatus, CreateSummaryArtifactRequest, EnsureThreadRequest, FilesystemSessionThreadService, FinalizedAssistantMessageByRunRequest, LoadContextMessagesRequest, LoadContextWindowRequest, MessageContent, MessageKind, @@ -206,6 +207,207 @@ async fn filesystem_finalized_assistant_lookup_by_run_uses_persisted_message() { assert_eq!(finalized.content.as_deref(), Some("final")); } +#[tokio::test] +async fn filesystem_append_finalized_assistant_message_is_finalized_and_idempotent_by_turn_run() { + let backend = Arc::new(InMemoryBackend::new()); + let scoped = scoped_threads_fs_at(backend, "tenant-finalized-append", "alice"); + let service = FilesystemSessionThreadService::new(scoped); + let scope = scope("finalized-append"); + let thread = service + .ensure_thread(EnsureThreadRequest { + scope: scope.clone(), + thread_id: Some(ThreadId::new("thread-finalized-append").unwrap()), + created_by_actor_id: "actor-a".into(), + title: None, + metadata_json: None, + }) + .await + .unwrap(); + + let first = service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized-append".into(), + content: MessageContent::text("final answer"), + }) + .await + .unwrap(); + let duplicate = service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized-append".into(), + content: MessageContent::text("retry answer ignored"), + }) + .await + .unwrap(); + + assert_eq!(first.message_id, duplicate.message_id); + assert_eq!(duplicate.kind, MessageKind::Assistant); + assert_eq!(duplicate.status, MessageStatus::Finalized); + assert_eq!(duplicate.content.as_deref(), Some("final answer")); + + let finalized = service + .finalized_assistant_message_by_run(FinalizedAssistantMessageByRunRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized-append".into(), + }) + .await + .unwrap() + .expect("finalized assistant message should be indexed by run"); + assert_eq!(finalized.message_id, first.message_id); + + let history = service + .list_thread_history(ThreadHistoryRequest { + scope, + thread_id: thread.thread_id, + }) + .await + .unwrap(); + assert_eq!(history.messages.len(), 1); + assert_eq!(history.messages[0].message_id, first.message_id); + assert_eq!(history.messages[0].status, MessageStatus::Finalized); +} + +#[tokio::test] +async fn filesystem_append_finalized_assistant_message_finalizes_existing_draft_by_turn_run() { + let backend = Arc::new(InMemoryBackend::new()); + let scoped = scoped_threads_fs_at(backend, "tenant-finalized-existing-draft", "alice"); + let service = FilesystemSessionThreadService::new(scoped); + let scope = scope("finalized-existing-draft"); + let thread = service + .ensure_thread(EnsureThreadRequest { + scope: scope.clone(), + thread_id: Some(ThreadId::new("thread-finalized-existing-draft").unwrap()), + created_by_actor_id: "actor-a".into(), + title: None, + metadata_json: None, + }) + .await + .unwrap(); + + let draft = service + .append_assistant_draft(AppendAssistantDraftRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized-existing-draft".into(), + content: MessageContent::text("draft answer"), + }) + .await + .unwrap(); + let finalized = service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized-existing-draft".into(), + content: MessageContent::text("final answer"), + }) + .await + .unwrap(); + + assert_eq!(finalized.message_id, draft.message_id); + assert_eq!(finalized.status, MessageStatus::Finalized); + assert_eq!(finalized.content.as_deref(), Some("final answer")); + + // The run index resolves to the same single message — finalizing in place + // must not leave the run pointing at a stale or second record. + let by_run = service + .finalized_assistant_message_by_run(FinalizedAssistantMessageByRunRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized-existing-draft".into(), + }) + .await + .unwrap() + .expect("finalized assistant message should be indexed by run"); + assert_eq!(by_run.message_id, draft.message_id); + assert_eq!(by_run.status, MessageStatus::Finalized); + + // Finalize-by-turn-run finalizes the existing draft IN PLACE — it must + // not materialize a second history row. Assert the caller-visible + // single-row invariant, not just the returned record. + let history = service + .list_thread_history(ThreadHistoryRequest { + scope, + thread_id: thread.thread_id, + }) + .await + .unwrap(); + assert_eq!(history.messages.len(), 1); + assert_eq!(history.messages[0].message_id, draft.message_id); + assert_eq!(history.messages[0].status, MessageStatus::Finalized); + assert_eq!(history.messages[0].content.as_deref(), Some("final answer")); +} + +#[tokio::test] +async fn filesystem_redacts_append_only_finalized_assistant_message() { + // Regression for the append-log mutation gap: a finalized assistant + // message written with NO prior draft lives only in the per-thread + // append log (no individual message file). Redaction must still apply — + // `apply_message_update` materializes the file on mutation and the + // file-authoritative merge then shadows the original log entry, so reads + // surface the redacted record (not the stale appended one) and history + // stays single-row. + let backend = Arc::new(InMemoryBackend::new()); + let scoped = scoped_threads_fs_at(backend, "tenant-redact-append-only", "alice"); + let service = FilesystemSessionThreadService::new(scoped); + let scope = scope("redact-append-only"); + let thread = service + .ensure_thread(EnsureThreadRequest { + scope: scope.clone(), + thread_id: Some(ThreadId::new("thread-redact-append-only").unwrap()), + created_by_actor_id: "actor-a".into(), + title: None, + metadata_json: None, + }) + .await + .unwrap(); + + // No prior draft -> this finalized message is append-only. + let finalized = service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-redact-append-only".into(), + content: MessageContent::text("secret answer"), + }) + .await + .unwrap(); + assert_eq!(finalized.status, MessageStatus::Finalized); + + let redacted = service + .redact_message(RedactMessageRequest { + scope: scope.clone(), + thread_id: thread.thread_id.clone(), + message_id: finalized.message_id, + redaction_ref: "redaction/audit/append-only".into(), + }) + .await + .unwrap(); + assert_eq!(redacted.status, MessageStatus::Redacted); + assert_eq!(redacted.content, None); + + // Reads must reflect the redaction, not the original append-log entry, + // and must not duplicate the message. + let history = service + .list_thread_history(ThreadHistoryRequest { + scope, + thread_id: thread.thread_id, + }) + .await + .unwrap(); + assert_eq!(history.messages.len(), 1); + assert_eq!(history.messages[0].message_id, finalized.message_id); + assert_eq!(history.messages[0].status, MessageStatus::Redacted); + assert_eq!(history.messages[0].content, None); + assert_eq!( + history.messages[0].redaction_ref.as_deref(), + Some("redaction/audit/append-only") + ); +} + #[tokio::test] async fn filesystem_lookup_index_write_failure_does_not_fail_message_contract() { let backend = Arc::new(LookupIndexWriteFailureBackend::new()); @@ -1318,7 +1520,7 @@ async fn filesystem_list_threads_orders_by_last_activity_not_creation() { // The freshly-used thread now leads the Recent list. let after = service .list_threads_for_scope(ListThreadsForScopeRequest { - scope: scope_a, + scope: scope_a.clone(), limit: None, cursor: None, }) @@ -1330,6 +1532,79 @@ async fn filesystem_list_threads_orders_by_last_activity_not_creation() { .map(|record| record.thread_id.as_str()) .collect(); assert_eq!(after_ids, ["t-older", "t-newer"]); + + // Cross-thread recency invariant the activity sort exists for: a + // *chattier but staler* thread must NOT outrank a *quieter but more + // recently touched* one. A per-thread-sequence sort (transcript length) + // would wrongly float `t-newer` above `t-older` after the steps below. + let older_stamp = service + .read_thread(ThreadHistoryRequest { + scope: scope_a.clone(), + thread_id: ThreadId::new("t-older").unwrap(), + }) + .await + .unwrap() + .updated_at + .expect("touched thread has activity stamp"); + wait_until_after(older_stamp).await; + + // Pile several messages onto `t-newer`, raising its per-thread sequence + // well above `t-older`'s — but at this earlier instant. + for i in 0..3 { + service + .accept_inbound_message(AcceptInboundMessageRequest { + scope: scope_a.clone(), + thread_id: ThreadId::new("t-newer").unwrap(), + actor_id: "actor-a".into(), + source_binding_id: Some(format!("binding-chatter-{i}")), + reply_target_binding_id: None, + external_event_id: Some(format!("event-chatter-{i}")), + content: MessageContent::text("chatter on the new thread"), + }) + .await + .unwrap(); + } + let newer_stamp = service + .read_thread(ThreadHistoryRequest { + scope: scope_a.clone(), + thread_id: ThreadId::new("t-newer").unwrap(), + }) + .await + .unwrap() + .updated_at + .expect("touched thread has activity stamp"); + wait_until_after(newer_stamp).await; + + // Touch `t-older` once more, strictly later. It now has FEWER total + // messages than `t-newer` but the most recent activity. + service + .accept_inbound_message(AcceptInboundMessageRequest { + scope: scope_a.clone(), + thread_id: ThreadId::new("t-older").unwrap(), + actor_id: "actor-a".into(), + source_binding_id: Some("binding-activity-2".into()), + reply_target_binding_id: None, + external_event_id: Some("event-activity-2".into()), + content: MessageContent::text("ping the old thread again"), + }) + .await + .unwrap(); + + let final_list = service + .list_threads_for_scope(ListThreadsForScopeRequest { + scope: scope_a, + limit: None, + cursor: None, + }) + .await + .unwrap(); + let final_ids: Vec<&str> = final_list + .threads + .iter() + .map(|record| record.thread_id.as_str()) + .collect(); + // Recency wins over transcript length. + assert_eq!(final_ids, ["t-older", "t-newer"]); } #[tokio::test] diff --git a/crates/ironclaw_threads/tests/session_thread_contract.rs b/crates/ironclaw_threads/tests/session_thread_contract.rs index c32d42bce2e..cef370ad19f 100644 --- a/crates/ironclaw_threads/tests/session_thread_contract.rs +++ b/crates/ironclaw_threads/tests/session_thread_contract.rs @@ -5,15 +5,17 @@ use ironclaw_host_api::{ }; use ironclaw_threads::{ AcceptInboundMessageRequest, AppendAssistantDraftRequest, - AppendCapabilityDisplayPreviewRequest, AppendToolResultReferenceRequest, AttachmentKind, - AttachmentRef, CapabilityDisplayPreviewEnvelope, CapabilityDisplayPreviewEnvelopeInput, + AppendCapabilityDisplayPreviewRequest, AppendFinalizedAssistantMessageRequest, + AppendToolResultReferenceRequest, AttachmentKind, AttachmentRef, + CapabilityDisplayPreviewEnvelope, CapabilityDisplayPreviewEnvelopeInput, CapabilityDisplayPreviewStatus, CreateSummaryArtifactRequest, EnsureThreadRequest, - InMemorySessionThreadService, ListThreadsForScopeRequest, LoadContextMessagesRequest, - LoadContextWindowRequest, MessageContent, MessageKind, MessageStatus, - ProviderToolCallReferenceEnvelope, RedactMessageRequest, SessionThreadError, - SessionThreadService, SummaryKind, SummaryModelContextPolicy, ThreadHistoryRequest, - ThreadMessageId, ThreadMessageRangeRequest, ThreadScope, ToolResultReferenceEnvelope, - ToolResultSafeSummary, UpdateAssistantDraftRequest, UpdateToolResultReferenceRequest, + FinalizedAssistantMessageByRunRequest, InMemorySessionThreadService, + ListThreadsForScopeRequest, LoadContextMessagesRequest, LoadContextWindowRequest, + MessageContent, MessageKind, MessageStatus, ProviderToolCallReferenceEnvelope, + RedactMessageRequest, SessionThreadError, SessionThreadService, SummaryKind, + SummaryModelContextPolicy, ThreadHistoryRequest, ThreadMessageId, ThreadMessageRangeRequest, + ThreadScope, ToolResultReferenceEnvelope, ToolResultSafeSummary, UpdateAssistantDraftRequest, + UpdateToolResultReferenceRequest, }; fn scope(label: &str) -> ThreadScope { @@ -2129,6 +2131,105 @@ async fn duplicate_assistant_draft_for_same_turn_run_is_idempotent() { assert_eq!(history.messages[0].status, MessageStatus::Redacted); } +#[tokio::test] +async fn append_finalized_assistant_message_is_finalized_and_idempotent_by_turn_run() { + let service = InMemorySessionThreadService::default(); + let thread = service + .ensure_thread(EnsureThreadRequest { + scope: scope("a"), + thread_id: None, + created_by_actor_id: "actor-a".into(), + title: None, + metadata_json: None, + }) + .await + .unwrap(); + + let first = service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: scope("a"), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized".into(), + content: MessageContent::text("final answer"), + }) + .await + .unwrap(); + let duplicate = service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: scope("a"), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized".into(), + content: MessageContent::text("retry answer ignored"), + }) + .await + .unwrap(); + + assert_eq!(first.message_id, duplicate.message_id); + assert_eq!(duplicate.kind, MessageKind::Assistant); + assert_eq!(duplicate.status, MessageStatus::Finalized); + assert_eq!(duplicate.content.as_deref(), Some("final answer")); + + let by_run = service + .finalized_assistant_message_by_run(FinalizedAssistantMessageByRunRequest { + scope: scope("a"), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-finalized".into(), + }) + .await + .unwrap() + .expect("finalized assistant message should be lookupable by run"); + assert_eq!(by_run.message_id, first.message_id); + + let history = service + .list_thread_history(ThreadHistoryRequest { + scope: scope("a"), + thread_id: thread.thread_id, + }) + .await + .unwrap(); + assert_eq!(history.messages.len(), 1); + assert_eq!(history.messages[0].message_id, first.message_id); + assert_eq!(history.messages[0].status, MessageStatus::Finalized); +} + +#[tokio::test] +async fn append_finalized_assistant_message_finalizes_existing_draft_by_turn_run() { + let service = InMemorySessionThreadService::default(); + let thread = service + .ensure_thread(EnsureThreadRequest { + scope: scope("a"), + thread_id: None, + created_by_actor_id: "actor-a".into(), + title: None, + metadata_json: None, + }) + .await + .unwrap(); + + let draft = service + .append_assistant_draft(AppendAssistantDraftRequest { + scope: scope("a"), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-existing-draft".into(), + content: MessageContent::text("draft answer"), + }) + .await + .unwrap(); + let finalized = service + .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { + scope: scope("a"), + thread_id: thread.thread_id.clone(), + turn_run_id: "run-existing-draft".into(), + content: MessageContent::text("final answer"), + }) + .await + .unwrap(); + + assert_eq!(finalized.message_id, draft.message_id); + assert_eq!(finalized.status, MessageStatus::Finalized); + assert_eq!(finalized.content.as_deref(), Some("final answer")); +} + #[tokio::test] async fn overlapping_replacement_summaries_are_rejected() { let service = InMemorySessionThreadService::default(); diff --git a/migrations/V32__root_filesystem_sequences.sql b/migrations/V32__root_filesystem_sequences.sql new file mode 100644 index 00000000000..2319ff07790 --- /dev/null +++ b/migrations/V32__root_filesystem_sequences.sql @@ -0,0 +1,11 @@ +-- Path-local monotonic sequence allocator for row-shaped stores. +-- +-- `root_filesystem_events` assigns globally increasing ids, which are correct +-- for event replay cursors but not for per-record-set ordering such as thread +-- message sequences. This table keeps one atomic counter per virtual path. + +CREATE TABLE IF NOT EXISTS root_filesystem_sequences ( + path TEXT PRIMARY KEY CHECK (path LIKE '/%'), + next_seq BIGINT NOT NULL CHECK (next_seq > 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/migrations/checksums.lock b/migrations/checksums.lock index e1f5c8d931e..8c4fa358ca7 100644 --- a/migrations/checksums.lock +++ b/migrations/checksums.lock @@ -44,3 +44,4 @@ V28__root_filesystem_records = 3989652150745692033 V29__root_filesystem_index_specs = 1572937030361291327 V30__root_filesystem_events = 7621479473639627207 V31__root_filesystem_path_collation = 14076261567902873431 +V32__root_filesystem_sequences = 10477775227430430324 diff --git a/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/README.md b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/README.md new file mode 100644 index 00000000000..d76f41d6c6f --- /dev/null +++ b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/README.md @@ -0,0 +1,100 @@ +# libSQL write-concurrency: WAL + storage-rework results — 2026-06-30 + +Measures the two-step throughput work on the libSQL `RootFilesystem` write path: + +1. **WAL + PRAGMA tuning** (PR #5451, merged) — `journal_mode=WAL`, + `synchronous=NORMAL`, plus cache/mmap/temp_store tuning. +2. **Row-native sequence primitive + thread/turn append paths** (PR #5455) — + `reserve_sequence` + finalized assistant-append, collapsing per-turn + full-document rewrites into appends. + +## Environment + +- Kernel: `Linux 6.18.5 x86_64` +- CPU: 4 cores (cloud container) +- Backend: `libsql` (local file) +- Build: `rustc 1.96.0`, `cargo build -p ironclaw_stress --release` +- Date: 2026-06-30 + +> NOTE: This is a 4-core Linux container, **not** the Apple M4 used by +> `2026-06-30-usable-boundary/`. Absolute numbers are not comparable across +> the two machines; use the before/after deltas *within this directory*, which +> were all collected on the same box. + +## Scenario + +Pure-storage `chat-turn` (no synthetic model wait), so latency reflects +storage only: + +```bash +cargo run -p ironclaw_stress --release -- \ + --backend libsql --scenario chat-turn \ + --sweep-concurrency --operations --users \ + --progress-interval-seconds 0 --output-jsonl +``` + +## Step 1 — WAL + PRAGMA tuning (#5451) + +`--operations 30 --users 200`. Aggregate p95 and throughput, before vs after. + +| Concurrency | p95 (DELETE journal) | p95 (WAL) | throughput (DELETE → WAL) | failures | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 168.9ms | 94.3ms | 12.9 → 11.2 ops/s | 0 / 0 | +| 4 | 1091.8ms | 217.8ms | 12.8 → 27.7 ops/s | 0 / 0 | +| 8 | 1285.5ms | 176.5ms | 9.4 → 33.6 ops/s | 0 / 0 | +| 16 | 1139.5ms | 199.6ms | 6.9 → 27.7 ops/s | 0 / 0 | +| 32 | 1311.7ms | 276.4ms | 5.7 → 19.5 ops/s | **2 → 0** | + +Without WAL, p95 jumps to ~1.1–1.3s the moment concurrency exceeds 1 and +throughput *declines* as concurrency rises (writers serialize destructively on +the whole-file lock, with lock-timeout failures at c32). With WAL, p95 holds at +175–280ms and throughput rises to a ~33 ops/s plateau. + +Artifacts: `wal-off-chatturn.jsonl`, `wal-on-chatturn.jsonl`. + +## Step 2 — storage rework on top of WAL (#5455) + +`--operations 30 --users 200`. WAL-only vs WAL + append-native storage. + +| Concurrency | `thread_store_writes` p95 | aggregate p95 | throughput | +| ---: | ---: | ---: | ---: | +| 8 | 126.2ms → 106.4ms | 176.5ms → 200.8ms | 33.6 → 36.2 ops/s | +| 32 | **186.0ms → 97.0ms** | **276.4ms → 183.6ms** | **19.5 → 34.2 ops/s** | + +WAL-only throughput collapses past c8 (19.5 ops/s at c32); with the +append-native paths it holds flat from c8 to c32. + +Artifacts: `wal-plus-rework-chatturn.jsonl` (compare against +`wal-on-chatturn.jsonl`). + +## Headline — 100 concurrent writes (WAL + storage rework) + +`--operations 20 --users 500`, sweeping to c100. + +| Concurrency | throughput | p50 | p95 | p99 | failures | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 8 | 31.9 ops/s | 103.3ms | 234.1ms | 432.9ms | 0 | +| 32 | 38.5 ops/s | 93.6ms | 174.4ms | 258.6ms | 0 | +| 64 | 30.5 ops/s | 124.2ms | 191.9ms | 227.0ms | 0 | +| **100** | 21.1 ops/s | 183.1ms | **256.8ms** | **294.3ms** | **0 / 2000** | + +**100 concurrent writes complete with zero failures and p95/p99 well inside +the usability SLO** (`p95 ≤ 2s`, `p99 ≤ 5s`). Throughput peaks at c32 and eases +to 21 ops/s at c100 — no collapse. The remaining bottleneck is split evenly +between `thread_store_writes` (~100–136ms) and `turn_store` (~91–143ms) — the +still-monolithic snapshot read-modify-writes; `context_reads` is negligible +(~12–22ms). + +Artifact: `storage-rework-chatturn-to-c100.jsonl`. + +## Interpretation + +- The journey: pre-WAL throughput *fell* with concurrency (failures at c32); + +WAL held ~33 ops/s but tapered past c8; +storage-rework holds with no + failures and bounded p95 to c100. +- `chat-turn` has no model latency. Real turns are model-bound, so storage + sustaining 100 concurrent at sub-300ms p95 means storage is no longer the + ceiling for realistic workloads. +- Next structural lever for the *throughput* ceiling (not concurrency/latency, + which are met): per-scope sharding of the turn/thread snapshots so + non-overlapping work stops contending on one document. diff --git a/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/storage-rework-chatturn-to-c100.jsonl b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/storage-rework-chatturn-to-c100.jsonl new file mode 100644 index 00000000000..7d227986b66 --- /dev/null +++ b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/storage-rework-chatturn-to-c100.jsonl @@ -0,0 +1,4 @@ +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":5244,"duration_seconds":0,"label":"r1 c8 u500 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":160,"cpu_ms":4804,"failed":0,"max_us":477029,"p95_us":234058,"p99_us":432946,"peak_rss_kb":22588,"throughput_ops_sec":31.935625016579205},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":20,"preset":null,"processes":1,"repetition":1,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c8-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"2abc76daef3242a693609a4c32713dea","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":160,"backend":"libsql","child_index":null,"concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":2797568,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":40960,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":2756608,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":5010,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":477029,"min_us":37839,"p50_us":103308,"p95_us":234058,"p99_us":432946},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":160,"latency":{"max_us":106751,"min_us":1406,"p50_us":3445,"p95_us":22107,"p99_us":81894}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":160,"latency":{"max_us":240001,"min_us":14308,"p50_us":46266,"p95_us":127483,"p99_us":223148}},"turn_store":{"count":160,"latency":{"max_us":331611,"min_us":9531,"p50_us":43267,"p95_us":135807,"p99_us":208634}}},"operations_per_thread":20,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":4804,"delta_system_cpu_ms":1943,"delta_user_cpu_ms":2861,"end":{"open_fds":11,"peak_rss_kb":22588,"rss_kb":17188,"system_cpu_ms":1946,"threads":6,"user_cpu_ms":2868},"peak_open_fds":21,"peak_rss_kb":22588,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":12032,"rss_kb":12232,"system_cpu_ms":3,"threads":6,"user_cpu_ms":7}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c8-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":160,"latency":{"max_us":227538,"min_us":3100,"p50_us":18678,"p95_us":80600,"p99_us":196835}},"append_assistant":{"count":160,"latency":{"max_us":103736,"min_us":1555,"p50_us":5934,"p95_us":36119,"p99_us":84795}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":160,"latency":{"max_us":109795,"min_us":2455,"p50_us":14330,"p95_us":49131,"p99_us":97038}},"complete_run":{"count":160,"latency":{"max_us":231634,"min_us":3165,"p50_us":18791,"p95_us":90673,"p99_us":190344}},"ensure_thread":{"count":160,"latency":{"max_us":84143,"min_us":1000,"p50_us":3997,"p95_us":24947,"p99_us":81314}},"finalize_assistant":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"load_context":{"count":160,"latency":{"max_us":106751,"min_us":1406,"p50_us":3445,"p95_us":22107,"p99_us":81894}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":160,"latency":{"max_us":86433,"min_us":985,"p50_us":4496,"p95_us":38655,"p99_us":84162}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":160,"latency":{"max_us":141875,"min_us":822,"p50_us":3828,"p95_us":43898,"p99_us":92612}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":160,"target":"libsql://","tenants":1,"throughput_ops_sec":31.935625016579205,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":16733,"duration_seconds":0,"label":"r1 c32 u500 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":640,"cpu_ms":27768,"failed":0,"max_us":389294,"p95_us":174387,"p99_us":258566,"peak_rss_kb":52368,"throughput_ops_sec":38.46969520904631},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":20,"preset":null,"processes":1,"repetition":1,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c32-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"2abc76daef3242a693609a4c32713dea","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":640,"backend":"libsql","child_index":null,"concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":13447168,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":2797568,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":10649600,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":16636,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":389294,"min_us":32076,"p50_us":93560,"p95_us":174387,"p99_us":258566},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":640,"latency":{"max_us":85855,"min_us":2305,"p50_us":4989,"p95_us":12394,"p99_us":36935}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":640,"latency":{"max_us":341013,"min_us":11254,"p50_us":41111,"p95_us":103932,"p99_us":155085}},"turn_store":{"count":640,"latency":{"max_us":263390,"min_us":12855,"p50_us":40886,"p95_us":90999,"p99_us":142767}}},"operations_per_thread":20,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":27768,"delta_system_cpu_ms":10567,"delta_user_cpu_ms":17201,"end":{"open_fds":11,"peak_rss_kb":52368,"rss_kb":17952,"system_cpu_ms":12513,"threads":5,"user_cpu_ms":20070},"peak_open_fds":22,"peak_rss_kb":52368,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":22588,"rss_kb":17364,"system_cpu_ms":1946,"threads":6,"user_cpu_ms":2869}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c32-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":640,"latency":{"max_us":197390,"min_us":3329,"p50_us":17329,"p95_us":55381,"p99_us":96407}},"append_assistant":{"count":640,"latency":{"max_us":245616,"min_us":2344,"p50_us":6136,"p95_us":30907,"p99_us":90185}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":640,"latency":{"max_us":143838,"min_us":4279,"p50_us":11383,"p95_us":43089,"p99_us":87996}},"complete_run":{"count":640,"latency":{"max_us":203217,"min_us":6798,"p50_us":18867,"p95_us":50903,"p99_us":73437}},"ensure_thread":{"count":640,"latency":{"max_us":112129,"min_us":201,"p50_us":3381,"p95_us":21457,"p99_us":38658}},"finalize_assistant":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"load_context":{"count":640,"latency":{"max_us":85855,"min_us":2305,"p50_us":4989,"p95_us":12394,"p99_us":36935}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":640,"latency":{"max_us":116007,"min_us":1749,"p50_us":4753,"p95_us":25623,"p99_us":46244}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":640,"latency":{"max_us":235105,"min_us":1319,"p50_us":4148,"p95_us":21962,"p99_us":40170}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":640,"target":"libsql://","tenants":1,"throughput_ops_sec":38.46969520904631,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":64,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":42015,"duration_seconds":0,"label":"r1 c64 u500 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":1280,"cpu_ms":84327,"failed":0,"max_us":292576,"p95_us":191914,"p99_us":227019,"peak_rss_kb":106812,"throughput_ops_sec":30.530657837813006},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":20,"preset":null,"processes":1,"repetition":1,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c64-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"2abc76daef3242a693609a4c32713dea","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":1280,"backend":"libsql","child_index":null,"concurrency":64,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":33304576,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":13447168,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":19857408,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":41925,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":292576,"min_us":52418,"p50_us":124182,"p95_us":191914,"p99_us":227019},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":1280,"latency":{"max_us":110572,"min_us":3919,"p50_us":8426,"p95_us":14970,"p99_us":29290}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":1280,"latency":{"max_us":200154,"min_us":16722,"p50_us":51297,"p95_us":100408,"p99_us":131309}},"turn_store":{"count":1280,"latency":{"max_us":224878,"min_us":24269,"p50_us":57423,"p95_us":105885,"p99_us":142329}}},"operations_per_thread":20,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":84327,"delta_system_cpu_ms":31909,"delta_user_cpu_ms":52418,"end":{"open_fds":11,"peak_rss_kb":106812,"rss_kb":18836,"system_cpu_ms":44423,"threads":5,"user_cpu_ms":72489},"peak_open_fds":22,"peak_rss_kb":106812,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":52368,"rss_kb":17784,"system_cpu_ms":12514,"threads":6,"user_cpu_ms":20071}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c64-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":1280,"latency":{"max_us":189894,"min_us":8124,"p50_us":24412,"p95_us":62748,"p99_us":99228}},"append_assistant":{"count":1280,"latency":{"max_us":117129,"min_us":3350,"p50_us":8124,"p95_us":38707,"p99_us":62305}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":1280,"latency":{"max_us":116257,"min_us":5654,"p50_us":16853,"p95_us":49955,"p99_us":87900}},"complete_run":{"count":1280,"latency":{"max_us":154163,"min_us":11883,"p50_us":27756,"p95_us":59586,"p99_us":89775}},"ensure_thread":{"count":1280,"latency":{"max_us":86840,"min_us":239,"p50_us":1527,"p95_us":15930,"p99_us":37213}},"finalize_assistant":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"load_context":{"count":1280,"latency":{"max_us":110572,"min_us":3919,"p50_us":8426,"p95_us":14970,"p99_us":29290}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":1280,"latency":{"max_us":92402,"min_us":2588,"p50_us":6054,"p95_us":26384,"p99_us":47167}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":1280,"latency":{"max_us":130742,"min_us":2709,"p50_us":5705,"p95_us":24931,"p99_us":47116}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":1280,"target":"libsql://","tenants":1,"throughput_ops_sec":30.530657837813006,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":100,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":94830,"duration_seconds":0,"label":"r1 c100 u500 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":2000,"cpu_ms":198531,"failed":0,"max_us":367019,"p95_us":256829,"p99_us":294273,"peak_rss_kb":187784,"throughput_ops_sec":21.098544364082354},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":20,"preset":null,"processes":1,"repetition":1,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c100-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"2abc76daef3242a693609a4c32713dea","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":2000,"backend":"libsql","child_index":null,"concurrency":100,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":63516672,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":33304576,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":30212096,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":94793,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":367019,"min_us":86438,"p50_us":183147,"p95_us":256829,"p99_us":294273},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":2000,"latency":{"max_us":92507,"min_us":7034,"p50_us":13350,"p95_us":18829,"p99_us":26466}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":2000,"latency":{"max_us":230817,"min_us":26999,"p50_us":78662,"p95_us":135682,"p99_us":168620}},"turn_store":{"count":2000,"latency":{"max_us":226183,"min_us":36155,"p50_us":86675,"p95_us":143195,"p99_us":175217}}},"operations_per_thread":20,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":198531,"delta_system_cpu_ms":76660,"delta_user_cpu_ms":121871,"end":{"open_fds":11,"peak_rss_kb":187784,"rss_kb":19392,"system_cpu_ms":121084,"threads":5,"user_cpu_ms":194362},"peak_open_fds":22,"peak_rss_kb":187784,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":106812,"rss_kb":18372,"system_cpu_ms":44424,"threads":6,"user_cpu_ms":72491}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"2abc76daef3242a693609a4c32713dea-r1-c100-u500-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":2000,"latency":{"max_us":143068,"min_us":12322,"p50_us":39484,"p95_us":88314,"p99_us":120485}},"append_assistant":{"count":2000,"latency":{"max_us":128509,"min_us":4605,"p50_us":11236,"p95_us":47869,"p99_us":68887}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":2000,"latency":{"max_us":149405,"min_us":8036,"p50_us":23858,"p95_us":66415,"p99_us":95844}},"complete_run":{"count":2000,"latency":{"max_us":140537,"min_us":21049,"p50_us":41631,"p95_us":83535,"p99_us":107274}},"ensure_thread":{"count":2000,"latency":{"max_us":65942,"min_us":247,"p50_us":1711,"p95_us":20698,"p99_us":42674}},"finalize_assistant":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"load_context":{"count":2000,"latency":{"max_us":92507,"min_us":7034,"p50_us":13350,"p95_us":18829,"p99_us":26466}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":2000,"latency":{"max_us":154864,"min_us":4047,"p50_us":8690,"p95_us":42200,"p99_us":64765}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":2000,"latency":{"max_us":135207,"min_us":3912,"p50_us":8774,"p95_us":42634,"p99_us":66854}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":2000,"target":"libsql://","tenants":1,"throughput_ops_sec":21.098544364082354,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":500,"warmup_seconds":0} diff --git a/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-off-chatturn.jsonl b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-off-chatturn.jsonl new file mode 100644 index 00000000000..b1343a2b328 --- /dev/null +++ b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-off-chatturn.jsonl @@ -0,0 +1,5 @@ +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":1,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":2447,"duration_seconds":0,"label":"r1 c1 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":30,"cpu_ms":605,"failed":0,"max_us":188031,"p95_us":168890,"p99_us":188031,"peak_rss_kb":15008,"throughput_ops_sec":12.898388300592563},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c1-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"001b5435f689400c8d83eb83c4e29997","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":30,"backend":"libsql","child_index":null,"concurrency":1,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":581632,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":32768,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":548864,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":2325,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":188031,"min_us":54722,"p50_us":68051,"p95_us":168890,"p99_us":188031},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":30,"latency":{"max_us":1274,"min_us":584,"p50_us":790,"p95_us":1088,"p99_us":1274}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":30,"latency":{"max_us":143884,"min_us":35350,"p50_us":43825,"p95_us":124177,"p99_us":143884}},"turn_store":{"count":30,"latency":{"max_us":56871,"min_us":18686,"p50_us":22721,"p95_us":44507,"p99_us":56871}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":605,"delta_system_cpu_ms":223,"delta_user_cpu_ms":382,"end":{"open_fds":11,"peak_rss_kb":14896,"rss_kb":15008,"system_cpu_ms":229,"threads":6,"user_cpu_ms":382},"peak_open_fds":14,"peak_rss_kb":15008,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":11924,"rss_kb":12132,"system_cpu_ms":6,"threads":6,"user_cpu_ms":0}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c1-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":30,"latency":{"max_us":19284,"min_us":11560,"p50_us":13945,"p95_us":17661,"p99_us":19284}},"append_assistant":{"count":30,"latency":{"max_us":56479,"min_us":11711,"p50_us":14581,"p95_us":38225,"p99_us":56479}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":30,"latency":{"max_us":10299,"min_us":6245,"p50_us":7734,"p95_us":8563,"p99_us":10299}},"complete_run":{"count":30,"latency":{"max_us":32308,"min_us":9123,"p50_us":10847,"p95_us":31935,"p99_us":32308}},"ensure_thread":{"count":30,"latency":{"max_us":27603,"min_us":2616,"p50_us":3474,"p95_us":10543,"p99_us":27603}},"finalize_assistant":{"count":30,"latency":{"max_us":87310,"min_us":5720,"p50_us":7130,"p95_us":46812,"p99_us":87310}},"load_context":{"count":30,"latency":{"max_us":1274,"min_us":584,"p50_us":790,"p95_us":1088,"p99_us":1274}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":30,"latency":{"max_us":4977,"min_us":3028,"p50_us":3597,"p95_us":4535,"p99_us":4977}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":30,"latency":{"max_us":34851,"min_us":3140,"p50_us":3961,"p95_us":4570,"p99_us":34851}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":30,"target":"libsql://","tenants":1,"throughput_ops_sec":12.898388300592563,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":4,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":9417,"duration_seconds":0,"label":"r1 c4 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":120,"cpu_ms":3197,"failed":0,"max_us":2196779,"p95_us":1091762,"p99_us":1983796,"peak_rss_kb":17816,"throughput_ops_sec":12.794574504012534},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c4-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"001b5435f689400c8d83eb83c4e29997","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":120,"backend":"libsql","child_index":null,"concurrency":4,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":2752512,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":581632,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":2170880,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":9378,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":2196779,"min_us":50545,"p50_us":136422,"p95_us":1091762,"p99_us":1983796},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":120,"latency":{"max_us":731945,"min_us":734,"p50_us":1458,"p95_us":105508,"p99_us":273930}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":120,"latency":{"max_us":2175199,"min_us":32104,"p50_us":61041,"p95_us":868729,"p99_us":1956489}},"turn_store":{"count":120,"latency":{"max_us":961024,"min_us":17362,"p50_us":24769,"p95_us":456665,"p99_us":947231}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":3197,"delta_system_cpu_ms":1372,"delta_user_cpu_ms":1825,"end":{"open_fds":11,"peak_rss_kb":17816,"rss_kb":16716,"system_cpu_ms":1601,"threads":6,"user_cpu_ms":2208},"peak_open_fds":17,"peak_rss_kb":17816,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":14896,"rss_kb":15104,"system_cpu_ms":229,"threads":6,"user_cpu_ms":383}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c4-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":120,"latency":{"max_us":2142038,"min_us":10250,"p50_us":14557,"p95_us":258869,"p99_us":662453}},"append_assistant":{"count":120,"latency":{"max_us":1560126,"min_us":10816,"p50_us":15615,"p95_us":321434,"p99_us":601746}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":120,"latency":{"max_us":943256,"min_us":5799,"p50_us":7870,"p95_us":170786,"p99_us":742702}},"complete_run":{"count":120,"latency":{"max_us":659617,"min_us":8230,"p50_us":11109,"p95_us":87760,"p99_us":232169}},"ensure_thread":{"count":120,"latency":{"max_us":434552,"min_us":2525,"p50_us":3595,"p95_us":165688,"p99_us":388563}},"finalize_assistant":{"count":120,"latency":{"max_us":972738,"min_us":5136,"p50_us":6892,"p95_us":70186,"p99_us":132704}},"load_context":{"count":120,"latency":{"max_us":731945,"min_us":734,"p50_us":1458,"p95_us":105508,"p99_us":273930}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":120,"latency":{"max_us":581954,"min_us":2590,"p50_us":3526,"p95_us":31981,"p99_us":152477}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":120,"latency":{"max_us":839116,"min_us":2797,"p50_us":3764,"p95_us":145952,"p99_us":436574}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":120,"target":"libsql://","tenants":1,"throughput_ops_sec":12.794574504012534,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":25647,"duration_seconds":0,"label":"r1 c8 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":240,"cpu_ms":10758,"failed":0,"max_us":1903706,"p95_us":1285492,"p99_us":1820237,"peak_rss_kb":19744,"throughput_ops_sec":9.388126363864165},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c8-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"001b5435f689400c8d83eb83c4e29997","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":240,"backend":"libsql","child_index":null,"concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":7143424,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":2752512,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":4390912,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":25564,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":1903706,"min_us":65767,"p50_us":297620,"p95_us":1285492,"p99_us":1820237},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":240,"latency":{"max_us":1050622,"min_us":1513,"p50_us":3108,"p95_us":195292,"p99_us":617403}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":240,"latency":{"max_us":1652049,"min_us":36220,"p50_us":170899,"p95_us":908924,"p99_us":1446396}},"turn_store":{"count":240,"latency":{"max_us":1509446,"min_us":20859,"p50_us":53155,"p95_us":321604,"p99_us":645379}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":10758,"delta_system_cpu_ms":4879,"delta_user_cpu_ms":5879,"end":{"open_fds":11,"peak_rss_kb":19692,"rss_kb":17040,"system_cpu_ms":6480,"threads":5,"user_cpu_ms":8087},"peak_open_fds":17,"peak_rss_kb":19744,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":17816,"rss_kb":16752,"system_cpu_ms":1601,"threads":6,"user_cpu_ms":2208}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c8-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":240,"latency":{"max_us":1167983,"min_us":11707,"p50_us":26222,"p95_us":287641,"p99_us":1054985}},"append_assistant":{"count":240,"latency":{"max_us":1474768,"min_us":12146,"p50_us":31873,"p95_us":248943,"p99_us":987281}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":240,"latency":{"max_us":1484589,"min_us":6394,"p50_us":10661,"p95_us":134745,"p99_us":539651}},"complete_run":{"count":240,"latency":{"max_us":855144,"min_us":10173,"p50_us":18321,"p95_us":171236,"p99_us":390056}},"ensure_thread":{"count":240,"latency":{"max_us":675505,"min_us":156,"p50_us":5209,"p95_us":234705,"p99_us":564394}},"finalize_assistant":{"count":240,"latency":{"max_us":645931,"min_us":5463,"p50_us":9613,"p95_us":140424,"p99_us":340198}},"load_context":{"count":240,"latency":{"max_us":1050622,"min_us":1513,"p50_us":3108,"p95_us":195292,"p99_us":617403}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":240,"latency":{"max_us":754687,"min_us":2797,"p50_us":4297,"p95_us":78130,"p99_us":625706}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":240,"latency":{"max_us":385524,"min_us":3027,"p50_us":4526,"p95_us":102197,"p99_us":334189}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":240,"target":"libsql://","tenants":1,"throughput_ops_sec":9.388126363864165,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":16,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":69631,"duration_seconds":0,"label":"r1 c16 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":480,"cpu_ms":37157,"failed":0,"max_us":3120560,"p95_us":1139498,"p99_us":1851074,"peak_rss_kb":22640,"throughput_ops_sec":6.899978061511129},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c16-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"001b5435f689400c8d83eb83c4e29997","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":480,"backend":"libsql","child_index":null,"concurrency":16,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":15659008,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":7143424,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":8515584,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":69565,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":3120560,"min_us":90475,"p50_us":514346,"p95_us":1139498,"p99_us":1851074},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":480,"latency":{"max_us":558034,"min_us":2911,"p50_us":8416,"p95_us":125120,"p99_us":335427}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":480,"latency":{"max_us":2444127,"min_us":56335,"p50_us":327843,"p95_us":842619,"p99_us":1269312}},"turn_store":{"count":480,"latency":{"max_us":2404050,"min_us":26746,"p50_us":120293,"p95_us":408684,"p99_us":739767}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":37157,"delta_system_cpu_ms":17856,"delta_user_cpu_ms":19301,"end":{"open_fds":11,"peak_rss_kb":22640,"rss_kb":17548,"system_cpu_ms":24337,"threads":5,"user_cpu_ms":27389},"peak_open_fds":17,"peak_rss_kb":22640,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":19692,"rss_kb":17032,"system_cpu_ms":6481,"threads":6,"user_cpu_ms":8088}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c16-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":480,"latency":{"max_us":950703,"min_us":14928,"p50_us":98640,"p95_us":334421,"p99_us":584274}},"append_assistant":{"count":480,"latency":{"max_us":1599497,"min_us":17183,"p50_us":95006,"p95_us":398083,"p99_us":795954}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":480,"latency":{"max_us":581187,"min_us":7919,"p50_us":22302,"p95_us":209558,"p99_us":438333}},"complete_run":{"count":480,"latency":{"max_us":2281900,"min_us":13060,"p50_us":39193,"p95_us":224183,"p99_us":668288}},"ensure_thread":{"count":480,"latency":{"max_us":2032866,"min_us":164,"p50_us":18839,"p95_us":145517,"p99_us":398548}},"finalize_assistant":{"count":480,"latency":{"max_us":668107,"min_us":7335,"p50_us":20048,"p95_us":175944,"p99_us":352859}},"load_context":{"count":480,"latency":{"max_us":558034,"min_us":2911,"p50_us":8416,"p95_us":125120,"p99_us":335427}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":480,"latency":{"max_us":2072406,"min_us":3681,"p50_us":5781,"p95_us":111527,"p99_us":364358}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":480,"latency":{"max_us":548813,"min_us":3820,"p50_us":7453,"p95_us":112728,"p99_us":279719}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":480,"target":"libsql://","tenants":1,"throughput_ops_sec":6.899978061511129,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":169121,"duration_seconds":0,"label":"r1 c32 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":960,"cpu_ms":111221,"failed":2,"max_us":63015985,"p95_us":1311669,"p99_us":1708105,"peak_rss_kb":26324,"throughput_ops_sec":5.679048492121547},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c32-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"001b5435f689400c8d83eb83c4e29997","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":960,"backend":"libsql","child_index":null,"concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":32555008,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":15659008,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":16896000,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":169042,"duration_seconds":0,"errors":{"turn_thread_busy":2},"failed":2,"failure_causes":{"turn_thread_busy":{"count":2,"sample_detail":"thread already has an active run","stages":{"submit_turn":2}}},"latency":{"max_us":63015985,"min_us":104841,"p50_us":644293,"p95_us":1311669,"p99_us":1708105},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":958,"latency":{"max_us":2278154,"min_us":5263,"p50_us":30796,"p95_us":155660,"p99_us":276603}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":960,"latency":{"max_us":2176480,"min_us":64894,"p50_us":391081,"p95_us":911771,"p99_us":1288339}},"turn_store":{"count":960,"latency":{"max_us":62118542,"min_us":353,"p50_us":159014,"p95_us":494762,"p99_us":855382}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":111221,"delta_system_cpu_ms":50111,"delta_user_cpu_ms":61110,"end":{"open_fds":11,"peak_rss_kb":26192,"rss_kb":18384,"system_cpu_ms":74449,"threads":5,"user_cpu_ms":88500},"peak_open_fds":17,"peak_rss_kb":26324,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":22640,"rss_kb":17404,"system_cpu_ms":24338,"threads":6,"user_cpu_ms":27390}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"001b5435f689400c8d83eb83c4e29997-r1-c32-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":960,"latency":{"max_us":1474689,"min_us":19102,"p50_us":108266,"p95_us":392104,"p99_us":690052}},"append_assistant":{"count":958,"latency":{"max_us":2050474,"min_us":22495,"p50_us":120903,"p95_us":390928,"p99_us":721721}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":958,"latency":{"max_us":827405,"min_us":10276,"p50_us":46453,"p95_us":196799,"p99_us":467528}},"complete_run":{"count":958,"latency":{"max_us":866181,"min_us":16956,"p50_us":65511,"p95_us":271517,"p99_us":493224}},"ensure_thread":{"count":960,"latency":{"max_us":1030770,"min_us":164,"p50_us":16548,"p95_us":145969,"p99_us":289109}},"finalize_assistant":{"count":958,"latency":{"max_us":1103696,"min_us":9594,"p50_us":38750,"p95_us":198234,"p99_us":464970}},"load_context":{"count":958,"latency":{"max_us":2278154,"min_us":5263,"p50_us":30796,"p95_us":155660,"p99_us":276603}},"mark_rejected_busy":{"count":2,"latency":{"max_us":52081,"min_us":20846,"p50_us":52081,"p95_us":52081,"p99_us":52081}},"mark_submitted":{"count":958,"latency":{"max_us":1474667,"min_us":4768,"p50_us":15225,"p95_us":116653,"p99_us":241129}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":960,"latency":{"max_us":61909377,"min_us":353,"p50_us":18956,"p95_us":137730,"p99_us":321702}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":958,"target":"libsql://","tenants":1,"throughput_ops_sec":5.679048492121547,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} diff --git a/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-on-chatturn.jsonl b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-on-chatturn.jsonl new file mode 100644 index 00000000000..b95e37b26e4 --- /dev/null +++ b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-on-chatturn.jsonl @@ -0,0 +1,5 @@ +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":1,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":2761,"duration_seconds":0,"label":"r1 c1 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":30,"cpu_ms":920,"failed":0,"max_us":94386,"p95_us":94267,"p99_us":94386,"peak_rss_kb":15404,"throughput_ops_sec":11.199591291971032},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c1-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"78454c45150e4e789c53a654499baf3f","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":30,"backend":"libsql","child_index":null,"concurrency":1,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":581632,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":32768,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":548864,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":2678,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":94386,"min_us":82461,"p50_us":89254,"p95_us":94267,"p99_us":94386},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":30,"latency":{"max_us":1527,"min_us":1192,"p50_us":1351,"p95_us":1525,"p99_us":1527}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":30,"latency":{"max_us":60059,"min_us":52363,"p50_us":56055,"p95_us":60018,"p99_us":60059}},"turn_store":{"count":30,"latency":{"max_us":35041,"min_us":28622,"p50_us":31646,"p95_us":33849,"p99_us":35041}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":920,"delta_system_cpu_ms":433,"delta_user_cpu_ms":487,"end":{"open_fds":11,"peak_rss_kb":15392,"rss_kb":15200,"system_cpu_ms":440,"threads":6,"user_cpu_ms":487},"peak_open_fds":15,"peak_rss_kb":15404,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":12088,"rss_kb":12320,"system_cpu_ms":7,"threads":6,"user_cpu_ms":0}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c1-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":30,"latency":{"max_us":20038,"min_us":16857,"p50_us":18454,"p95_us":19265,"p99_us":20038}},"append_assistant":{"count":30,"latency":{"max_us":21279,"min_us":17568,"p50_us":18582,"p95_us":20847,"p99_us":21279}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":30,"latency":{"max_us":12068,"min_us":9838,"p50_us":10928,"p95_us":11857,"p99_us":12068}},"complete_run":{"count":30,"latency":{"max_us":18333,"min_us":13103,"p50_us":14790,"p95_us":17947,"p99_us":18333}},"ensure_thread":{"count":30,"latency":{"max_us":5768,"min_us":4190,"p50_us":4614,"p95_us":5309,"p99_us":5768}},"finalize_assistant":{"count":30,"latency":{"max_us":11863,"min_us":8739,"p50_us":9317,"p95_us":10522,"p99_us":11863}},"load_context":{"count":30,"latency":{"max_us":1527,"min_us":1192,"p50_us":1351,"p95_us":1525,"p99_us":1527}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":30,"latency":{"max_us":6400,"min_us":4508,"p50_us":4895,"p95_us":5133,"p99_us":6400}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":30,"latency":{"max_us":6248,"min_us":5392,"p50_us":5680,"p95_us":6207,"p99_us":6248}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":30,"target":"libsql://","tenants":1,"throughput_ops_sec":11.199591291971032,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":4,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":4409,"duration_seconds":0,"label":"r1 c4 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":120,"cpu_ms":5011,"failed":0,"max_us":398273,"p95_us":217819,"p99_us":323344,"peak_rss_kb":24532,"throughput_ops_sec":27.700827814908692},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c4-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"78454c45150e4e789c53a654499baf3f","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":120,"backend":"libsql","child_index":null,"concurrency":4,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":2813952,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":581632,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":2232320,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":4332,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":398273,"min_us":62708,"p50_us":133240,"p95_us":217819,"p99_us":323344},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":120,"latency":{"max_us":83878,"min_us":1088,"p50_us":2895,"p95_us":21272,"p99_us":83083}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":120,"latency":{"max_us":246329,"min_us":38216,"p50_us":73025,"p95_us":158643,"p99_us":241563}},"turn_store":{"count":120,"latency":{"max_us":174741,"min_us":15964,"p50_us":39719,"p95_us":110811,"p99_us":163271}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":5011,"delta_system_cpu_ms":2252,"delta_user_cpu_ms":2759,"end":{"open_fds":11,"peak_rss_kb":24532,"rss_kb":16748,"system_cpu_ms":2693,"threads":6,"user_cpu_ms":3246},"peak_open_fds":21,"peak_rss_kb":24532,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":15392,"rss_kb":15320,"system_cpu_ms":441,"threads":6,"user_cpu_ms":487}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c4-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":120,"latency":{"max_us":117923,"min_us":4819,"p50_us":18836,"p95_us":69519,"p99_us":112453}},"append_assistant":{"count":120,"latency":{"max_us":141244,"min_us":6607,"p50_us":20405,"p95_us":72324,"p99_us":100775}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":120,"latency":{"max_us":111065,"min_us":3633,"p50_us":11494,"p95_us":58054,"p99_us":79414}},"complete_run":{"count":120,"latency":{"max_us":109491,"min_us":6876,"p50_us":16048,"p95_us":72374,"p99_us":101371}},"ensure_thread":{"count":120,"latency":{"max_us":56358,"min_us":791,"p50_us":3490,"p95_us":21334,"p99_us":39515}},"finalize_assistant":{"count":120,"latency":{"max_us":118827,"min_us":2885,"p50_us":10232,"p95_us":86826,"p99_us":116057}},"load_context":{"count":120,"latency":{"max_us":83878,"min_us":1088,"p50_us":2895,"p95_us":21272,"p99_us":83083}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":120,"latency":{"max_us":39412,"min_us":1473,"p50_us":4812,"p95_us":24158,"p99_us":37742}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":120,"latency":{"max_us":58865,"min_us":1535,"p50_us":3857,"p95_us":36535,"p99_us":57677}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":120,"target":"libsql://","tenants":1,"throughput_ops_sec":27.700827814908692,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":7217,"duration_seconds":0,"label":"r1 c8 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":240,"cpu_ms":11952,"failed":0,"max_us":291638,"p95_us":176462,"p99_us":248718,"peak_rss_kb":40552,"throughput_ops_sec":33.60123964784623},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c8-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"78454c45150e4e789c53a654499baf3f","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":240,"backend":"libsql","child_index":null,"concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":7344128,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":2813952,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":4530176,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":7142,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":291638,"min_us":60301,"p50_us":106003,"p95_us":176462,"p99_us":248718},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":240,"latency":{"max_us":22976,"min_us":1676,"p50_us":3394,"p95_us":8168,"p99_us":13193}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":240,"latency":{"max_us":246450,"min_us":29972,"p50_us":63240,"p95_us":126248,"p99_us":176196}},"turn_store":{"count":240,"latency":{"max_us":106951,"min_us":14773,"p50_us":37933,"p95_us":79205,"p99_us":100942}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":11952,"delta_system_cpu_ms":5229,"delta_user_cpu_ms":6723,"end":{"open_fds":11,"peak_rss_kb":40552,"rss_kb":17308,"system_cpu_ms":7923,"threads":6,"user_cpu_ms":9969},"peak_open_fds":23,"peak_rss_kb":40552,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":24532,"rss_kb":16784,"system_cpu_ms":2694,"threads":6,"user_cpu_ms":3246}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c8-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":240,"latency":{"max_us":193737,"min_us":7092,"p50_us":15275,"p95_us":55469,"p99_us":96618}},"append_assistant":{"count":240,"latency":{"max_us":128998,"min_us":9835,"p50_us":20437,"p95_us":62808,"p99_us":98057}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":240,"latency":{"max_us":88622,"min_us":3853,"p50_us":8881,"p95_us":34389,"p99_us":61422}},"complete_run":{"count":240,"latency":{"max_us":82246,"min_us":4614,"p50_us":17897,"p95_us":46930,"p99_us":67108}},"ensure_thread":{"count":240,"latency":{"max_us":57021,"min_us":186,"p50_us":3101,"p95_us":15346,"p99_us":35789}},"finalize_assistant":{"count":240,"latency":{"max_us":85567,"min_us":3286,"p50_us":7911,"p95_us":31425,"p99_us":61696}},"load_context":{"count":240,"latency":{"max_us":22976,"min_us":1676,"p50_us":3394,"p95_us":8168,"p99_us":13193}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":240,"latency":{"max_us":83873,"min_us":1687,"p50_us":3833,"p95_us":21880,"p99_us":26052}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":240,"latency":{"max_us":82935,"min_us":1635,"p50_us":3782,"p95_us":21512,"p99_us":39235}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":240,"target":"libsql://","tenants":1,"throughput_ops_sec":33.60123964784623,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":16,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":17351,"duration_seconds":0,"label":"r1 c16 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":480,"cpu_ms":33046,"failed":0,"max_us":279791,"p95_us":199551,"p99_us":247191,"peak_rss_kb":66560,"throughput_ops_sec":27.672289455408883},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c16-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"78454c45150e4e789c53a654499baf3f","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":480,"backend":"libsql","child_index":null,"concurrency":16,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":15851520,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":7344128,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":8507392,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":17345,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":279791,"min_us":69638,"p50_us":139676,"p95_us":199551,"p99_us":247191},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":480,"latency":{"max_us":39215,"min_us":3118,"p50_us":5320,"p95_us":10009,"p99_us":23115}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":480,"latency":{"max_us":207662,"min_us":37828,"p50_us":81545,"p95_us":131341,"p99_us":164849}},"turn_store":{"count":480,"latency":{"max_us":190945,"min_us":20854,"p50_us":45687,"p95_us":88523,"p99_us":114441}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":33046,"delta_system_cpu_ms":14719,"delta_user_cpu_ms":18327,"end":{"open_fds":11,"peak_rss_kb":66560,"rss_kb":17664,"system_cpu_ms":22643,"threads":5,"user_cpu_ms":28297},"peak_open_fds":22,"peak_rss_kb":66560,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":40552,"rss_kb":17296,"system_cpu_ms":7924,"threads":6,"user_cpu_ms":9970}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c16-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":480,"latency":{"max_us":140512,"min_us":6789,"p50_us":21721,"p95_us":52494,"p99_us":81042}},"append_assistant":{"count":480,"latency":{"max_us":184771,"min_us":12296,"p50_us":27617,"p95_us":67019,"p99_us":88020}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":480,"latency":{"max_us":84904,"min_us":4947,"p50_us":10942,"p95_us":36617,"p99_us":60951}},"complete_run":{"count":480,"latency":{"max_us":164117,"min_us":10991,"p50_us":24041,"p95_us":53493,"p99_us":78904}},"ensure_thread":{"count":480,"latency":{"max_us":45543,"min_us":225,"p50_us":1689,"p95_us":13874,"p99_us":25069}},"finalize_assistant":{"count":480,"latency":{"max_us":77960,"min_us":4190,"p50_us":10371,"p95_us":38365,"p99_us":62696}},"load_context":{"count":480,"latency":{"max_us":39215,"min_us":3118,"p50_us":5320,"p95_us":10009,"p99_us":23115}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":480,"latency":{"max_us":73854,"min_us":2201,"p50_us":4564,"p95_us":24686,"p99_us":41886}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":480,"latency":{"max_us":59947,"min_us":2142,"p50_us":4391,"p95_us":24555,"p99_us":40395}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":480,"target":"libsql://","tenants":1,"throughput_ops_sec":27.672289455408883,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":49269,"duration_seconds":0,"label":"r1 c32 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":960,"cpu_ms":99786,"failed":0,"max_us":576848,"p95_us":276350,"p99_us":341829,"peak_rss_kb":112444,"throughput_ops_sec":19.499275866421424},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c32-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"78454c45150e4e789c53a654499baf3f","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":960,"backend":"libsql","child_index":null,"concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":32800768,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":15851520,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":16949248,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":49232,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":576848,"min_us":100036,"p50_us":196228,"p95_us":276350,"p99_us":341829},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":960,"latency":{"max_us":65770,"min_us":4670,"p50_us":8815,"p95_us":13119,"p99_us":22114}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":960,"latency":{"max_us":469989,"min_us":52263,"p50_us":111731,"p95_us":185965,"p99_us":222946}},"turn_store":{"count":960,"latency":{"max_us":192783,"min_us":26846,"p50_us":69665,"p95_us":124417,"p99_us":151321}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":99786,"delta_system_cpu_ms":42006,"delta_user_cpu_ms":57780,"end":{"open_fds":11,"peak_rss_kb":112444,"rss_kb":18396,"system_cpu_ms":64650,"threads":5,"user_cpu_ms":86078},"peak_open_fds":23,"peak_rss_kb":112444,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":66560,"rss_kb":17512,"system_cpu_ms":22644,"threads":6,"user_cpu_ms":28298}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"78454c45150e4e789c53a654499baf3f-r1-c32-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":960,"latency":{"max_us":149963,"min_us":13247,"p50_us":32554,"p95_us":78964,"p99_us":105253}},"append_assistant":{"count":960,"latency":{"max_us":381576,"min_us":16239,"p50_us":41911,"p95_us":91261,"p99_us":124233}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":960,"latency":{"max_us":143863,"min_us":6787,"p50_us":15529,"p95_us":53229,"p99_us":76181}},"complete_run":{"count":960,"latency":{"max_us":155705,"min_us":15662,"p50_us":35247,"p95_us":73599,"p99_us":100106}},"ensure_thread":{"count":960,"latency":{"max_us":62047,"min_us":243,"p50_us":1334,"p95_us":12482,"p99_us":34017}},"finalize_assistant":{"count":960,"latency":{"max_us":100372,"min_us":6667,"p50_us":13901,"p95_us":49400,"p99_us":69879}},"load_context":{"count":960,"latency":{"max_us":65770,"min_us":4670,"p50_us":8815,"p95_us":13119,"p99_us":22114}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":960,"latency":{"max_us":335752,"min_us":3111,"p50_us":6250,"p95_us":29745,"p99_us":64221}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":960,"latency":{"max_us":101883,"min_us":2977,"p50_us":6491,"p95_us":33280,"p99_us":61610}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":960,"target":"libsql://","tenants":1,"throughput_ops_sec":19.499275866421424,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} diff --git a/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-plus-rework-chatturn.jsonl b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-plus-rework-chatturn.jsonl new file mode 100644 index 00000000000..04ee156b20a --- /dev/null +++ b/tools/ironclaw_stress/results/2026-06-30-wal-and-storage-rework/wal-plus-rework-chatturn.jsonl @@ -0,0 +1,2 @@ +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":6726,"duration_seconds":0,"label":"r1 c8 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":240,"cpu_ms":8040,"failed":0,"max_us":314930,"p95_us":200814,"p99_us":287990,"peak_rss_kb":24412,"throughput_ops_sec":36.19318404242606},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"f44ef491996d46eea99f8eb25b571d05-r1-c8-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"f44ef491996d46eea99f8eb25b571d05","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":240,"backend":"libsql","child_index":null,"concurrency":8,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":4083712,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":40960,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":4042752,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":6631,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":314930,"min_us":37734,"p50_us":95577,"p95_us":200814,"p99_us":287990},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":240,"latency":{"max_us":82724,"min_us":993,"p50_us":4156,"p95_us":22300,"p99_us":58386}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":240,"latency":{"max_us":228745,"min_us":15731,"p50_us":39183,"p95_us":106369,"p99_us":150962}},"turn_store":{"count":240,"latency":{"max_us":219503,"min_us":10198,"p50_us":40215,"p95_us":110725,"p99_us":156776}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":8040,"delta_system_cpu_ms":3920,"delta_user_cpu_ms":4120,"end":{"open_fds":11,"peak_rss_kb":24412,"rss_kb":17164,"system_cpu_ms":3923,"threads":6,"user_cpu_ms":4123},"peak_open_fds":21,"peak_rss_kb":24412,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":11944,"rss_kb":12220,"system_cpu_ms":3,"threads":6,"user_cpu_ms":3}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"f44ef491996d46eea99f8eb25b571d05-r1-c8-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":240,"latency":{"max_us":127449,"min_us":5845,"p50_us":16744,"p95_us":55275,"p99_us":103005}},"append_assistant":{"count":240,"latency":{"max_us":111229,"min_us":1156,"p50_us":6190,"p95_us":42140,"p99_us":83480}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":240,"latency":{"max_us":123365,"min_us":2001,"p50_us":12360,"p95_us":60906,"p99_us":110242}},"complete_run":{"count":240,"latency":{"max_us":120065,"min_us":2246,"p50_us":16945,"p95_us":56098,"p99_us":92872}},"ensure_thread":{"count":240,"latency":{"max_us":59233,"min_us":249,"p50_us":3388,"p95_us":22570,"p99_us":37698}},"finalize_assistant":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"load_context":{"count":240,"latency":{"max_us":82724,"min_us":993,"p50_us":4156,"p95_us":22300,"p99_us":58386}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":240,"latency":{"max_us":109384,"min_us":1115,"p50_us":5211,"p95_us":25471,"p99_us":60128}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":240,"latency":{"max_us":108190,"min_us":1035,"p50_us":4631,"p95_us":26793,"p99_us":58314}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":240,"target":"libsql://","tenants":1,"throughput_ops_sec":36.19318404242606,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} +{"active_thread_count":0,"assistant_message_bytes":0,"backend":"libsql","concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"duration_ms":28162,"duration_seconds":0,"label":"r1 c32 u200 at0 m0 ub0 ab0 ctx20 cg4 tc2 tb1024","metrics":{"attempted":960,"cpu_ms":47410,"failed":0,"max_us":440654,"p95_us":183604,"p99_us":229776,"peak_rss_kb":55112,"throughput_ops_sec":34.17757597028514},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_spike_every":0,"model_latency_spike_ms":0,"operations_per_thread":30,"preset":null,"processes":1,"repetition":1,"run_id":"f44ef491996d46eea99f8eb25b571d05-r1-c32-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","suite_run_id":"f44ef491996d46eea99f8eb25b571d05","summary":{"active_thread_count":0,"assistant_message_bytes":0,"attempted":960,"backend":"libsql","child_index":null,"concurrency":32,"context_growth_turns_per_operation":4,"context_max_messages":20,"db_probe":{"after":{"libsql_file_bytes":18935808,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"before":{"libsql_file_bytes":4083712,"libsql_shm_bytes":0,"libsql_wal_bytes":0},"delta":{"libsql_file_bytes":14852096,"libsql_shm_bytes":0,"libsql_wal_bytes":0}},"duration_ms":28088,"duration_seconds":0,"errors":{},"failed":0,"latency":{"max_us":440654,"min_us":40683,"p50_us":108293,"p95_us":183604,"p99_us":229776},"model_latency_jitter_ms":0,"model_latency_ms":0,"model_latency_profile":"fixed","model_latency_source":"synthetic","model_latency_spike_every":0,"model_latency_spike_ms":0,"operation_attribution":{"context_reads":{"count":960,"latency":{"max_us":60630,"min_us":2128,"p50_us":5610,"p95_us":12532,"p99_us":25389}},"resource_governor":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"synthetic_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"thread_store_writes":{"count":960,"latency":{"max_us":393391,"min_us":12862,"p50_us":43859,"p95_us":97013,"p99_us":129913}},"turn_store":{"count":960,"latency":{"max_us":242372,"min_us":20470,"p50_us":53388,"p95_us":103675,"p99_us":151759}}},"operations_per_thread":30,"prefill_concurrency":4,"prefill_threads":0,"prefill_turns_per_thread":0,"preset":null,"process":{"delta_cpu_ms":47410,"delta_system_cpu_ms":23067,"delta_user_cpu_ms":24343,"end":{"open_fds":11,"peak_rss_kb":55112,"rss_kb":18268,"system_cpu_ms":26991,"threads":5,"user_cpu_ms":28467},"peak_open_fds":22,"peak_rss_kb":55112,"peak_threads":7,"start":{"open_fds":11,"peak_rss_kb":24412,"rss_kb":17264,"system_cpu_ms":3924,"threads":6,"user_cpu_ms":4124}},"processes":1,"provider_max_tokens":16,"provider_model":null,"run_id":"f44ef491996d46eea99f8eb25b571d05-r1-c32-u200-at0-m0-ub0-ab0-ctx20-cg4-tc2-tb1024","scenario":"chat-turn","stage_latency":{"accept_inbound":{"count":960,"latency":{"max_us":383322,"min_us":5123,"p50_us":19580,"p95_us":56995,"p99_us":88277}},"append_assistant":{"count":960,"latency":{"max_us":110638,"min_us":1860,"p50_us":7436,"p95_us":39882,"p99_us":78717}},"append_tool_preview":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"append_tool_result":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"claim_run":{"count":960,"latency":{"max_us":163966,"min_us":3966,"p50_us":14119,"p95_us":46803,"p99_us":81929}},"complete_run":{"count":960,"latency":{"max_us":177294,"min_us":5928,"p50_us":25644,"p95_us":61873,"p99_us":93510}},"ensure_thread":{"count":960,"latency":{"max_us":60259,"min_us":193,"p50_us":911,"p95_us":10365,"p99_us":24647}},"finalize_assistant":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"load_context":{"count":960,"latency":{"max_us":60630,"min_us":2128,"p50_us":5610,"p95_us":12532,"p99_us":25389}},"mark_rejected_busy":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"mark_submitted":{"count":960,"latency":{"max_us":338100,"min_us":1692,"p50_us":5287,"p95_us":31358,"p99_us":62019}},"model_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reconcile":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_release":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"resource_reserve":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"submit_turn":{"count":960,"latency":{"max_us":112402,"min_us":2088,"p50_us":5455,"p95_us":26496,"p99_us":42716}},"tool_wait":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}},"update_assistant_draft":{"count":0,"latency":{"max_us":0,"min_us":0,"p50_us":0,"p95_us":0,"p99_us":0}}},"succeeded":960,"target":"libsql://","tenants":1,"throughput_ops_sec":34.17757597028514,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0},"tenants":1,"tool_calls_per_turn":2,"tool_failure_every":0,"tool_latency_ms":0,"tool_output_bytes":1024,"trace_interval_seconds":1,"trace_jsonl":null,"trace_jsonl_enabled":false,"user_message_bytes":0,"users":200,"warmup_seconds":0} diff --git a/tools/ironclaw_stress/src/user_turn.rs b/tools/ironclaw_stress/src/user_turn.rs index 2fb8e939d6a..da5b658dc4f 100644 --- a/tools/ironclaw_stress/src/user_turn.rs +++ b/tools/ironclaw_stress/src/user_turn.rs @@ -21,11 +21,11 @@ use ironclaw_llm::{ use ironclaw_resources::{ResourceError, ResourceGovernor}; use ironclaw_threads::{ AcceptInboundMessageRequest, AppendAssistantDraftRequest, - AppendCapabilityDisplayPreviewRequest, AppendToolResultReferenceRequest, - CapabilityDisplayPreviewEnvelope, CapabilityDisplayPreviewEnvelopeInput, - CapabilityDisplayPreviewStatus, EnsureThreadRequest, FilesystemSessionThreadService, - LoadContextWindowRequest, MessageContent, SessionThreadError, SessionThreadService, - ToolResultSafeSummary, UpdateAssistantDraftRequest, + AppendCapabilityDisplayPreviewRequest, AppendFinalizedAssistantMessageRequest, + AppendToolResultReferenceRequest, CapabilityDisplayPreviewEnvelope, + CapabilityDisplayPreviewEnvelopeInput, CapabilityDisplayPreviewStatus, EnsureThreadRequest, + FilesystemSessionThreadService, LoadContextWindowRequest, MessageContent, SessionThreadError, + SessionThreadService, ToolResultSafeSummary, UpdateAssistantDraftRequest, }; use ironclaw_turns::{ AcceptedMessageRef, DefaultTurnCoordinator, FilesystemTurnStateStore, IdempotencyKey, @@ -1126,30 +1126,19 @@ where assistant_message: String, stages: &mut UserTurnStageDurations, ) -> Result<(), OperationFailure> { - let draft = time_stage( + time_stage( &mut stages.append_assistant, - self.thread_service - .append_assistant_draft(AppendAssistantDraftRequest { + self.thread_service.append_finalized_assistant_message( + AppendFinalizedAssistantMessageRequest { scope: context.thread_scope.clone(), thread_id: thread_id.clone(), turn_run_id: claimed.state.run_id.to_string(), - content: MessageContent::text(assistant_message.clone()), - }), - ) - .await - .map_err(|error| thread_failure("append_assistant", error))?; - - time_stage( - &mut stages.finalize_assistant, - self.thread_service.finalize_assistant_message( - &context.thread_scope, - thread_id, - draft.message_id, - MessageContent::text(assistant_message), + content: MessageContent::text(assistant_message), + }, ), ) .await - .map_err(|error| thread_failure("finalize_assistant", error))?; + .map_err(|error| thread_failure("append_assistant", error))?; time_stage( &mut stages.complete_run, @@ -1251,9 +1240,8 @@ async fn reserve_resources( governor: Arc, scope: ResourceScope, ) -> Result { - tokio::task::spawn_blocking(move || governor.reserve(scope, resource_ops::estimate())) - .await - .map_err(|error| OperationFailure::new("resource_worker", "resource_reserve", error))? + governor + .reserve(scope, resource_ops::estimate()) .map_err(|error| resource_failure("resource_reserve", error)) } @@ -1261,9 +1249,8 @@ async fn reconcile_resources( governor: Arc, reservation_id: ResourceReservationId, ) -> Result<(), OperationFailure> { - tokio::task::spawn_blocking(move || governor.reconcile(reservation_id, resource_ops::usage())) - .await - .map_err(|error| OperationFailure::new("resource_worker", "resource_reconcile", error))? + governor + .reconcile(reservation_id, resource_ops::usage()) .map(|_| ()) .map_err(|error| resource_failure("resource_reconcile", error)) } @@ -1272,9 +1259,8 @@ async fn release_resources( governor: Arc, reservation_id: ResourceReservationId, ) -> Result<(), OperationFailure> { - tokio::task::spawn_blocking(move || governor.release(reservation_id)) - .await - .map_err(|error| OperationFailure::new("resource_worker", "resource_release", error))? + governor + .release(reservation_id) .map(|_| ()) .map_err(|error| resource_failure("resource_release", error)) }