From 1dee1f2cc8949b77bbac69a1394e7cea7095044a Mon Sep 17 00:00:00 2001 From: lawrencecchen Date: Thu, 27 Aug 2026 14:14:02 -0700 Subject: [PATCH 001/133] test(tui): require agent hook journal events to drive agent records Running an agent (claude, codex, ...) inside cmux-tui appends agent.* journal events through the installed hooks, but nothing turned those events into agent records, so every agents sidebar view stayed empty unless something called agent report directly. Red: committed hook events must update the terminal's record (idle/working/blocked/done), replays must not rewrite it, and terminal-less events must still append. --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c598484c4f8b..da06bee67f72 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -21498,6 +21498,98 @@ mod tests { assert_eq!(batches[1].changes[0]["value"]["state"], "blocked"); } + #[test] + fn committed_agent_hook_events_drive_the_terminal_agent_record() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let surface_id = surface.id; + let terminal_id = mux.with_state(|state| { + match state.resource_indexes.content_ids.get(&surface_id).unwrap() { + ContentPublicId::Terminal(terminal_id) => terminal_id.clone(), + ContentPublicId::Browser(_) => panic!("workspace opened a browser"), + } + }); + + let append = |event: &str, key: &str| { + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + event, + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"native-1"}), + ) + .unwrap(); + mux.append_journal_ingress(&ingress, "test", key).unwrap(); + }; + + append("SessionStart", "hook-1"); + let records = mux.list_agents(Some(surface_id), None); + assert_eq!(records.len(), 1); + assert_eq!(records[0].state, AgentState::Idle); + assert_eq!(records[0].source, AgentSource::Hook); + + append("UserPromptSubmit", "hook-2"); + assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Working); + + append("PermissionRequest", "hook-3"); + assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Blocked); + + append("Stop", "hook-4"); + assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Idle); + + // Child-agent events carry no top-level lifecycle transition. + append("SubagentStart", "hook-5"); + assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Idle); + + append("SessionEnd", "hook-6"); + assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Done); + + // A socket report cannot downgrade a hook-owned record. + mux.report_agent(surface_id, AgentState::Working, AgentSource::Socket, None).unwrap(); + assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Done); + } + + #[test] + fn replayed_agent_hook_events_do_not_rewrite_the_record() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let surface_id = surface.id; + let terminal_id = mux.with_state(|state| { + match state.resource_indexes.content_ids.get(&surface_id).unwrap() { + ContentPublicId::Terminal(terminal_id) => terminal_id.clone(), + ContentPublicId::Browser(_) => panic!("workspace opened a browser"), + } + }); + let started = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"native-1"}), + ) + .unwrap(); + mux.append_journal_ingress(&started, "test", "hook-replay").unwrap(); + let working_at = mux.list_agents(Some(surface_id), None)[0].updated_at_ms; + + let replay = mux.append_journal_ingress(&started, "test", "hook-replay").unwrap(); + assert!(replay.replayed); + let record = &mux.list_agents(Some(surface_id), None)[0]; + assert_eq!(record.state, AgentState::Working); + assert_eq!(record.updated_at_ms, working_at); + } + + #[test] + fn agent_hook_events_without_a_live_terminal_still_append() { + let mux = test_mux(); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + None, + serde_json::json!({}), + ) + .unwrap(); + mux.append_journal_ingress(&ingress, "test", "hook-no-terminal").unwrap(); + assert!(mux.list_agents(None, None).is_empty()); + } + #[test] fn failed_raw_agent_report_rolls_back_projection_memory_revision_and_event() { let mux = test_mux(); From b8b9b3841e10e5c4b00a103751f3f7bda3dadef7 Mon Sep 17 00:00:00 2001 From: lawrencecchen Date: Thu, 27 Aug 2026 14:14:03 -0700 Subject: [PATCH 002/133] fix(tui): feed agent records from committed hook journal events Every fresh (non-replayed) agent.* journal commit now updates its terminal's agent record inside Mux::append_journal_ingress, covering both the hook helper's socket path and the agent hook emit CLI. Mapping: session.started and turn.completed -> idle, turn.started -> working, approval/question/plan_review/error -> blocked, session.ended -> done; child events and unclassified state changes leave the record alone. Best effort: a hook may outlive its terminal, so record failures log instead of failing the append. --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 75 ++++++++++++++++++++---- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index da06bee67f72..853d9a7ac010 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1086,6 +1086,24 @@ impl AgentSource { } } +/// The agent-record state a committed hook journal event implies, or `None` +/// for events that carry no lifecycle transition (child agents, unclassified +/// state changes) so they never churn the record. +fn agent_state_for_hook_kind(kind: &str) -> Option { + Some(match kind { + // A freshly started session sits at its prompt; a completed turn + // returns to it. + "agent.session.started" | "agent.turn.completed" => AgentState::Idle, + "agent.turn.started" => AgentState::Working, + "agent.approval.requested" + | "agent.question.requested" + | "agent.plan_review.requested" + | "agent.error.reported" => AgentState::Blocked, + "agent.session.ended" => AgentState::Done, + _ => return None, + }) +} + #[derive(Debug, Clone)] pub struct AgentRecord { pub surface: SurfaceId, @@ -5148,26 +5166,61 @@ impl Mux { idempotency_key: &str, ) -> anyhow::Result { let validated = self.journal_kernel.validate_ingress(ingress)?; - if self.journal_ingress.enabled() { - return self.journal_ingress.send_producer( + let commit = if self.journal_ingress.enabled() { + self.journal_ingress.send_producer( ingress.clone(), validated, origin.into(), idempotency_key.into(), - ); - } - let commit = self.workspace_registry.lock().unwrap().append_journal_ingress( - ingress, - &validated, - origin, - idempotency_key, - )?; + )? + } else { + let commit = self.workspace_registry.lock().unwrap().append_journal_ingress( + ingress, + &validated, + origin, + idempotency_key, + )?; + if !commit.replayed { + self.publish_journal_event(); + } + commit + }; if !commit.replayed { - self.publish_journal_event(); + self.apply_agent_hook_record(ingress); } Ok(commit) } + /// Committed agent hook events double as the live agent-status feed: + /// a fresh `agent.*` journal event updates its terminal's agent record, + /// so agents views show working/blocked/idle/done without a separate + /// reporting channel. Best effort by design: a hook may outlive its + /// terminal (exit races, replays into closed tabs), and the journal + /// append must never start failing because the record cannot update. + fn apply_agent_hook_record(&self, ingress: &crate::JournalIngress) { + if ingress.producer_id != crate::agent_hooks::AGENT_HOOK_PRODUCER_ID { + return; + } + let Some(state) = agent_state_for_hook_kind(&ingress.kind) else { return }; + let Some(terminal_id) = ingress + .subjects + .iter() + .find(|subject| subject.kind == "terminal") + .and_then(|subject| TerminalPublicId::parse(&subject.id).ok()) + else { + return; + }; + let Some(surface) = self.resource_surface_for_terminal(&terminal_id) else { return }; + // The record's session field is a human-facing label; native agent + // session ids are opaque, so views fall back to their own context. + if let Err(error) = self.report_agent(surface, state, AgentSource::Hook, None) { + eprintln!( + "cmux-tui: agent record update for {} ({}) failed: {error}", + terminal_id, ingress.kind + ); + } + } + pub(crate) fn journal_hook_states( &self, ) -> anyhow::Result> { From 6ecbde8415b7e5472c92de6f6f034173d14dca19 Mon Sep 17 00:00:00 2001 From: lawrencecchen Date: Thu, 27 Aug 2026 15:37:37 -0700 Subject: [PATCH 003/133] fix(tui): drop the agent record when its session ends Exiting an agent should remove it from agents views, not leave a done row. The reducer still commits and broadcasts the done state first so remote caches converge, then drops the live record so a fresh agent in the same terminal starts clean. Terminal close already purged records; this covers the agent exiting while its shell stays open. --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 853d9a7ac010..a6c06f3a7fc4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5218,6 +5218,16 @@ impl Mux { "cmux-tui: agent record update for {} ({}) failed: {error}", terminal_id, ingress.kind ); + return; + } + // An ended session leaves the roster entirely: the done state was + // committed and broadcast above (so remote caches converge), and the + // live record is dropped so agents views stop listing the terminal + // and a fresh agent there starts clean. Hooks of one terminal are + // sequential (they follow one agent process's lifecycle), so nothing + // races this removal. + if state == AgentState::Done { + self.agent_records.lock().unwrap().remove(&terminal_id); } } @@ -21593,12 +21603,17 @@ mod tests { append("SubagentStart", "hook-5"); assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Idle); - append("SessionEnd", "hook-6"); - assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Done); - // A socket report cannot downgrade a hook-owned record. mux.report_agent(surface_id, AgentState::Working, AgentSource::Socket, None).unwrap(); - assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Done); + assert_eq!(mux.list_agents(Some(surface_id), None)[0].state, AgentState::Idle); + + // An exited agent leaves the roster; a fresh one starts clean. + append("SessionEnd", "hook-6"); + assert!(mux.list_agents(Some(surface_id), None).is_empty()); + append("SessionStart", "hook-7"); + let records = mux.list_agents(Some(surface_id), None); + assert_eq!(records.len(), 1); + assert_eq!(records[0].state, AgentState::Idle); } #[test] From fce4a34e106534ab163cbf8bfc29f687134633ab Mon Sep 17 00:00:00 2001 From: lawrencecchen Date: Thu, 27 Aug 2026 19:16:18 -0700 Subject: [PATCH 004/133] feat(tui): derive the agent roster from the session journal The live agents roster is now a journal reducer: a pure fold over committed agent.* records with a durable cursor and snapshot persisted in the registry meta table. Restart restores the snapshot and folds only the journal tail; wiping the snapshot (or bumping the reducer version) re-folds from the journal head to the identical state, which the tests prove. The fold reads each record back as the journal stored it, so live folds and replays are the same computation. Direct socket/SDK agent reports keep their synchronous projection commit and replay contract, and now also append an echo journal event carrying the committed state and timestamp, so the log sees every agent intent and the roster has exactly one write path. Hook-beats- socket precedence is arbitrated on the durable projection row under the commit's registry lock (serialized), and expressed identically in the fold; a done projection no longer pins precedence so a fresh agent in the same terminal starts clean. The restored-projection roster cache is deleted; closed terminals retire their roster entry and persist the snapshot. Each fresh direct report now publishes twice on the shared change epoch (resource commit plus journal echo); affected tests document that. --- cmux-tui/Cargo.lock | 2 +- .../crates/cmux-tui-core/src/agent_hooks.rs | 2 +- .../cmux-tui-core/src/journal_reducers.rs | 353 +++++++++++++ cmux-tui/crates/cmux-tui-core/src/lib.rs | 1 + cmux-tui/crates/cmux-tui-core/src/mux.rs | 488 +++++++++++++++--- .../src/mux/public_projections.rs | 43 +- cmux-tui/crates/cmux-tui-core/src/server.rs | 4 +- .../src/workspace_registry/session_journal.rs | 51 ++ 8 files changed, 830 insertions(+), 114 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 859ffecf1abd..725bf09eb3d8 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -760,8 +760,8 @@ dependencies = [ "cmux-tui-core", "cmux-tui-machine-agent-protocol", "cmux-tui-machine-protocol", - "crossterm", "crossbeam-channel", + "crossterm", "fs4", "getrandom 0.3.4", "ghostty-vt", diff --git a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs index 75842b6046f6..645076cccbae 100644 --- a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs +++ b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs @@ -9,7 +9,7 @@ use crate::{ pub const AGENT_HOOK_PRODUCER_ID: &str = "cmux_agent"; pub const AGENT_HOOK_MANIFEST_VERSION: u32 = 1; -const AGENT_HOOK_FORMAT: &str = "cmux.agent-hook.v1"; +pub(crate) const AGENT_HOOK_FORMAT: &str = "cmux.agent-hook.v1"; const MAX_AGENT_SOURCE_BYTES: usize = 64; const MAX_NATIVE_EVENT_BYTES: usize = 128; const NORMALIZED_TEXT_BYTES: usize = 8 * 1024; diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs new file mode 100644 index 000000000000..a59eb54cc66a --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -0,0 +1,353 @@ +//! Durable materialized views folded from the session journal. +//! +//! A journal reducer is a pure fold: committed `agent.*` records go in, a +//! deterministic state comes out. The daemon persists each reducer's cursor +//! (the last journal sequence folded) and a snapshot of its state in the +//! registry's `meta` table, so a restart restores the snapshot and re-folds +//! only the journal tail. Bumping a reducer's version discards the snapshot +//! and re-folds from the journal head; state older than the earliest +//! retained journal record is rebuilt lazily by the next events, which is +//! acceptable for live rosters and wrong for ledgers - keep ledger-shaped +//! reducers versioned conservatively. +//! +//! The first reducer is the agent roster: the set of live agents per +//! terminal that agents views render. Every write path is a journal event +//! (hook helpers append `agent.*` events; socket `agent report` appends an +//! echo event after its direct projection commit), so the roster never has +//! a second writer to diverge from. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::agent_hooks::AGENT_HOOK_PRODUCER_ID; +use crate::workspace_registry::SessionJournalRecord; +use crate::{AgentSource, AgentState, JournalSubject}; + +pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; +/// Bump to discard persisted snapshots and re-fold from the journal head. +pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 1; + +/// The adapter id and native event the socket report path uses for its echo +/// journal events. The echo carries the explicit state in `normalized`, so +/// the fold never has to guess a semantic mapping for it. +pub(crate) const SOCKET_REPORT_ADAPTER: &str = "socket"; +pub(crate) const SOCKET_REPORT_NATIVE_EVENT: &str = "StateReport"; + +fn agent_state_from_str(value: &str) -> Option { + Some(match value { + "working" => AgentState::Working, + "blocked" => AgentState::Blocked, + "idle" => AgentState::Idle, + "done" => AgentState::Done, + "unknown" => AgentState::Unknown, + _ => return None, + }) +} + +fn agent_source_from_str(value: &str) -> Option { + Some(match value { + "detected" => AgentSource::Detected, + "socket" => AgentSource::Socket, + "hook" => AgentSource::Hook, + _ => return None, + }) +} + +/// The lifecycle state a hook journal kind implies, or `None` for kinds +/// that carry no top-level transition (child agents, unclassified changes). +fn state_for_hook_kind(kind: &str) -> Option { + Some(match kind { + // A freshly started session sits at its prompt; a completed turn + // returns to it. + "agent.session.started" | "agent.turn.completed" => AgentState::Idle, + "agent.turn.started" => AgentState::Working, + "agent.approval.requested" + | "agent.question.requested" + | "agent.plan_review.requested" + | "agent.error.reported" => AgentState::Blocked, + "agent.session.ended" => AgentState::Done, + _ => return None, + }) +} + +/// One journal record reduced to the fields the roster fold reads. Built +/// from a live `JournalIngress` at commit time and from stored +/// `SessionJournalRecord`s during tail replay, with identical semantics so +/// both paths fold to the same state. +pub(crate) struct RosterEvent<'a> { + pub(crate) producer_id: &'a str, + pub(crate) kind: &'a str, + pub(crate) subjects: &'a [JournalSubject], + pub(crate) payload: &'a Value, + pub(crate) committed_at_ms: u64, +} + +impl<'a> RosterEvent<'a> { + pub(crate) fn from_record(record: &'a SessionJournalRecord) -> Self { + Self { + producer_id: &record.producer.id, + kind: &record.kind, + subjects: &record.subjects, + payload: &record.payload, + committed_at_ms: record.committed_at_ms, + } + } + + fn terminal_id(&self) -> Option<&str> { + self.subjects + .iter() + .find(|subject| subject.kind == "terminal") + .map(|subject| subject.id.as_str()) + } + + fn adapter_id(&self) -> Option<&str> { + self.payload.get("adapter")?.get("id")?.as_str() + } + + fn normalized(&self, field: &str) -> Option<&str> { + self.payload.get("normalized")?.get(field)?.as_str() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct RosterEntry { + pub(crate) state: String, + pub(crate) source: String, + pub(crate) session: Option, + pub(crate) updated_at_ms: u64, +} + +impl RosterEntry { + pub(crate) fn agent_state(&self) -> AgentState { + agent_state_from_str(&self.state).unwrap_or(AgentState::Unknown) + } + + pub(crate) fn agent_source(&self) -> AgentSource { + agent_source_from_str(&self.source).unwrap_or(AgentSource::Hook) + } +} + +/// A roster change produced by folding one record. The host applies these +/// as side effects (projection commits, change broadcasts); the fold itself +/// only mutates roster state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum RosterDelta { + Upsert { terminal_id: String, entry: RosterEntry }, + Remove { terminal_id: String }, +} + +/// Live-agent roster: terminal public id to the agent's last reported +/// lifecycle state. An ended session leaves the roster (history stays in +/// the journal and the durable agent projection); a hook-owned entry +/// ignores socket reports so a slow poller cannot overwrite live hook +/// state. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub(crate) struct AgentRoster { + pub(crate) entries: HashMap, +} + +impl AgentRoster { + /// Fold one committed record. Deterministic: identical event sequences + /// produce identical rosters, so a snapshot plus the journal tail always + /// reproduces the live state. + pub(crate) fn apply(&mut self, event: &RosterEvent<'_>) -> Vec { + if event.producer_id != AGENT_HOOK_PRODUCER_ID { + return Vec::new(); + } + let Some(terminal_id) = event.terminal_id() else { return Vec::new() }; + let (state, source, session, updated_at_ms) = if event.adapter_id() + == Some(SOCKET_REPORT_ADAPTER) + { + // Socket echo: explicit state and timestamp carried in the + // payload, so the roster mirrors the direct projection + // commit exactly. + let Some(state) = event.normalized("state").and_then(agent_state_from_str) else { + return Vec::new(); + }; + let source = event + .normalized("source") + .and_then(agent_source_from_str) + .unwrap_or(AgentSource::Socket); + let updated_at_ms = event + .normalized("updated_at_ms") + .and_then(|value| value.parse::().ok()) + .unwrap_or(event.committed_at_ms); + (state, source, event.normalized("source_session").map(str::to_string), updated_at_ms) + } else { + let Some(state) = state_for_hook_kind(event.kind) else { return Vec::new() }; + (state, AgentSource::Hook, None, event.committed_at_ms) + }; + if source == AgentSource::Socket + && self + .entries + .get(terminal_id) + .is_some_and(|entry| entry.agent_source() == AgentSource::Hook) + { + // Hook state is live agent truth; socket reports cannot + // overwrite it (mirrors the projection commit precedence). + return Vec::new(); + } + if state == AgentState::Done { + // An ended agent leaves the roster entirely; the done state is + // still committed to the durable projection by the host so + // history and remote caches converge. + return if self.entries.remove(terminal_id).is_some() { + vec![RosterDelta::Remove { terminal_id: terminal_id.to_string() }] + } else { + Vec::new() + }; + } + let entry = RosterEntry { + state: state.as_str().to_string(), + source: source.as_str().to_string(), + session, + updated_at_ms, + }; + if self.entries.get(terminal_id) == Some(&entry) { + return Vec::new(); + } + self.entries.insert(terminal_id.to_string(), entry.clone()); + vec![RosterDelta::Upsert { terminal_id: terminal_id.to_string(), entry }] + } + + /// Drop a terminal that left the session (tab closed, terminal + /// tombstoned). Terminal lifecycle does not flow through `agent.*` + /// events yet, so the host retires entries explicitly. + pub(crate) fn retire_terminal(&mut self, terminal_id: &str) -> bool { + self.entries.remove(terminal_id).is_some() + } + + pub(crate) fn snapshot(&self) -> Value { + serde_json::to_value(self).unwrap_or(Value::Null) + } + + pub(crate) fn restore(snapshot: &str) -> Option { + serde_json::from_str(snapshot).ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn hook_event<'a>( + sequence: u64, + kind: &'a str, + subjects: &'a [JournalSubject], + payload: &'a Value, + ) -> RosterEvent<'a> { + RosterEvent { + producer_id: AGENT_HOOK_PRODUCER_ID, + kind, + subjects, + payload, + committed_at_ms: 1_000 + sequence, + } + } + + fn terminal_subject(id: &str) -> Vec { + vec![JournalSubject { kind: "terminal".into(), id: id.into() }] + } + + #[test] + fn lifecycle_kinds_fold_into_roster_states() { + let subjects = terminal_subject("term_a"); + let payload = json!({}); + let mut roster = AgentRoster::default(); + + roster.apply(&hook_event(1, "agent.session.started", &subjects, &payload)); + assert_eq!(roster.entries["term_a"].state, "idle"); + + roster.apply(&hook_event(2, "agent.turn.started", &subjects, &payload)); + assert_eq!(roster.entries["term_a"].state, "working"); + + roster.apply(&hook_event(3, "agent.approval.requested", &subjects, &payload)); + assert_eq!(roster.entries["term_a"].state, "blocked"); + + // Child events carry no top-level transition. + let deltas = roster.apply(&hook_event(4, "agent.child.spawned", &subjects, &payload)); + assert!(deltas.is_empty()); + assert_eq!(roster.entries["term_a"].state, "blocked"); + + let deltas = roster.apply(&hook_event(5, "agent.session.ended", &subjects, &payload)); + assert_eq!(deltas, vec![RosterDelta::Remove { terminal_id: "term_a".into() }]); + assert!(roster.entries.is_empty()); + } + + #[test] + fn socket_echo_carries_explicit_state_and_loses_to_hook_entries() { + let subjects = terminal_subject("term_a"); + let socket_payload = json!({ + "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, + "normalized": {"state": "working", "source": "socket", "source_session": "probe"}, + }); + let mut roster = AgentRoster::default(); + + roster.apply(&hook_event(1, "agent.state.changed", &subjects, &socket_payload)); + let entry = &roster.entries["term_a"]; + assert_eq!(entry.state, "working"); + assert_eq!(entry.source, "socket"); + assert_eq!(entry.session.as_deref(), Some("probe")); + + // A hook event takes the terminal over... + let hook_payload = json!({}); + roster.apply(&hook_event(2, "agent.turn.started", &subjects, &hook_payload)); + assert_eq!(roster.entries["term_a"].source, "hook"); + + // ...and later socket reports cannot downgrade it. + let deltas = + roster.apply(&hook_event(3, "agent.state.changed", &subjects, &socket_payload)); + assert!(deltas.is_empty()); + assert_eq!(roster.entries["term_a"].source, "hook"); + } + + #[test] + fn folds_are_idempotent_per_state_and_deterministic_across_replays() { + let subjects = terminal_subject("term_a"); + let payload = json!({}); + let events = [ + "agent.session.started", + "agent.turn.started", + "agent.turn.started", + "agent.turn.completed", + ]; + + let mut live = AgentRoster::default(); + let mut delta_count = 0usize; + for (index, kind) in events.iter().enumerate() { + delta_count += + live.apply(&hook_event(index as u64 + 1, kind, &subjects, &payload)).len(); + } + // A same-state re-report still refreshes recency (chronological + // views sort on it), so every event here produces a delta. + assert_eq!(delta_count, 4); + + let mut replayed = AgentRoster::default(); + for (index, kind) in events.iter().enumerate() { + replayed.apply(&hook_event(index as u64 + 1, kind, &subjects, &payload)); + } + assert_eq!(serde_json::to_value(&live).unwrap(), serde_json::to_value(&replayed).unwrap()); + } + + #[test] + fn snapshot_round_trips_and_foreign_producers_are_ignored() { + let subjects = terminal_subject("term_a"); + let payload = json!({}); + let mut roster = AgentRoster::default(); + roster.apply(&hook_event(1, "agent.turn.started", &subjects, &payload)); + + let snapshot = roster.snapshot().to_string(); + let restored = AgentRoster::restore(&snapshot).unwrap(); + assert_eq!(restored.entries, roster.entries); + + let foreign = RosterEvent { + producer_id: "someone_else", + ..hook_event(2, "agent.session.ended", &subjects, &payload) + }; + assert!(roster.apply(&foreign).is_empty()); + assert!(!roster.entries.is_empty()); + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index 26ca50f53f87..58d8f0dd907b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -16,6 +16,7 @@ mod journal_checkpoint; mod journal_hooks; mod journal_ingress; mod journal_kernel; +mod journal_reducers; mod model; mod mux; mod pairing; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a6c06f3a7fc4..5fecb685e62d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1086,22 +1086,65 @@ impl AgentSource { } } -/// The agent-record state a committed hook journal event implies, or `None` -/// for events that carry no lifecycle transition (child agents, unclassified -/// state changes) so they never churn the record. -fn agent_state_for_hook_kind(kind: &str) -> Option { - Some(match kind { - // A freshly started session sits at its prompt; a completed turn - // returns to it. - "agent.session.started" | "agent.turn.completed" => AgentState::Idle, - "agent.turn.started" => AgentState::Working, - "agent.approval.requested" - | "agent.question.requested" - | "agent.plan_review.requested" - | "agent.error.reported" => AgentState::Blocked, - "agent.session.ended" => AgentState::Done, - _ => return None, - }) +/// A stored projection state string as its typed form; unknown spellings +/// degrade to `Unknown`, which every agents view hides. +fn parse_projection_agent_state(value: &str) -> AgentState { + match value { + "working" => AgentState::Working, + "blocked" => AgentState::Blocked, + "idle" => AgentState::Idle, + "done" => AgentState::Done, + _ => AgentState::Unknown, + } +} + +/// The agent roster host: reducer state plus its journal fold cursor. +/// Lock ordering rule: never acquire another `Mux` lock while holding this +/// one - fold paths release it before persisting, and commit paths only +/// take a read after their registry/state locks, so `registry -> roster` +/// is the single global order. +#[derive(Debug, Default)] +struct AgentRosterHost { + roster: crate::journal_reducers::AgentRoster, + cursor: u64, +} + +/// Restore the roster from its persisted snapshot and fold the journal tail +/// committed after the cursor. A reducer-version mismatch discards the +/// snapshot and re-folds from the journal head. Deltas produced here are +/// dropped deliberately: their projection commits and change broadcasts +/// already happened when the events first committed, and the durable +/// projection restores itself independently. +fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { + use crate::journal_reducers::{ + AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, AgentRoster, RosterEvent, + }; + let mut host = match registry.journal_reducer_state(AGENT_ROSTER_REDUCER_ID)? { + Some((version, cursor, snapshot)) if version == AGENT_ROSTER_REDUCER_VERSION => { + AgentRosterHost { roster: AgentRoster::restore(&snapshot).unwrap_or_default(), cursor } + } + _ => AgentRosterHost::default(), + }; + let started_at = host.cursor; + loop { + let page = registry.session_journal_after(host.cursor, 512)?; + if page.records.is_empty() { + break; + } + for record in &page.records { + host.roster.apply(&RosterEvent::from_record(record)); + host.cursor = host.cursor.max(record.sequence); + } + } + if host.cursor != started_at { + registry.put_journal_reducer_state( + AGENT_ROSTER_REDUCER_ID, + AGENT_ROSTER_REDUCER_VERSION, + host.cursor, + &host.roster.snapshot().to_string(), + )?; + } + Ok(host) } #[derive(Debug, Clone)] @@ -1122,6 +1165,16 @@ struct TerminalAgentRecord { updated_at_ms: u64, } +/// Who initiated an agent projection commit: a direct socket/SDK report +/// (which must echo its intent into the journal so the roster fold sees +/// it), or the roster fold itself applying a journal-derived delta (which +/// must not echo, or every hook event would append a second record). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AgentReportOrigin { + Direct, + RosterFold, +} + enum AgentReportTarget<'a> { Surface(SurfaceId), Resource { selectors: &'a crate::ResourceSelectors, terminal_id: &'a TerminalPublicId }, @@ -2024,7 +2077,7 @@ pub struct Mux { default_colors: Mutex, durable_terminal_defaults: AtomicBool, sidebar_plugin: Mutex, - agent_records: Mutex>, + agent_roster: Mutex, /// Nonterminal notifications remain placement-local. Terminal unread /// state is keyed separately by stable content identity so every view of /// one terminal shares the same attention marker. @@ -2292,10 +2345,10 @@ impl Mux { default_colors, has_terminal_defaults, next_notification_id, - agent_records, terminal_notifications, notification_ledger, } = restore_public_projections(&state, registry.public_projections()?)?; + let agent_roster = restore_agent_roster(®istry)?; let journal_producers = registry.journal_producer_manifests()?; let session_public_id = registry.session_id().clone(); let journal_kernel = crate::journal_kernel::JournalKernel::new( @@ -2401,7 +2454,7 @@ impl Mux { default_colors: Mutex::new(default_colors), durable_terminal_defaults: AtomicBool::new(has_terminal_defaults), sidebar_plugin: Mutex::new(SidebarPluginRuntime::default()), - agent_records: Mutex::new(agent_records), + agent_roster: Mutex::new(agent_roster), placement_notifications: Mutex::new(HashMap::new()), terminal_notifications: Mutex::new(terminal_notifications), notification_ledger: Mutex::new(notification_ledger), @@ -5186,48 +5239,177 @@ impl Mux { commit }; if !commit.replayed { - self.apply_agent_hook_record(ingress); + self.fold_agent_roster(ingress, &commit); } Ok(commit) } - /// Committed agent hook events double as the live agent-status feed: - /// a fresh `agent.*` journal event updates its terminal's agent record, - /// so agents views show working/blocked/idle/done without a separate - /// reporting channel. Best effort by design: a hook may outlive its - /// terminal (exit races, replays into closed tabs), and the journal - /// append must never start failing because the record cannot update. - fn apply_agent_hook_record(&self, ingress: &crate::JournalIngress) { + /// Fold one fresh `agent.*` journal commit into the roster reducer and + /// apply the resulting deltas (projection commits, change broadcasts). + /// The roster is derived state: this fold plus the startup tail replay + /// are its only writers, so the journal fully determines it. Best + /// effort by design: a hook may outlive its terminal, and a journal + /// append must never start failing because a view cannot update. + fn fold_agent_roster( + &self, + ingress: &crate::JournalIngress, + commit: &crate::JournalAppendCommit, + ) { + use crate::journal_reducers::{ + AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, RosterEvent, + SOCKET_REPORT_ADAPTER, + }; if ingress.producer_id != crate::agent_hooks::AGENT_HOOK_PRODUCER_ID { return; } - let Some(state) = agent_state_for_hook_kind(&ingress.kind) else { return }; - let Some(terminal_id) = ingress - .subjects - .iter() - .find(|subject| subject.kind == "terminal") - .and_then(|subject| TerminalPublicId::parse(&subject.id).ok()) - else { + // Fold the record as the journal stored it, not the ingress the + // caller sent: the stored row carries the committed timestamp and + // is the exact input the startup tail replay folds, so live folds + // and replays are identical by construction. + let record = match self + .workspace_registry + .lock() + .unwrap() + .session_journal_after(commit.sequence.saturating_sub(1), 1) + { + Ok(page) => { + match page.records.into_iter().find(|record| record.sequence == commit.sequence) { + Some(record) => record, + None => return, + } + } + Err(error) => { + eprintln!("cmux-tui: reading a committed agent event back failed: {error}"); + return; + } + }; + let (deltas, cursor, snapshot) = { + let mut host = self.agent_roster.lock().unwrap(); + if commit.sequence <= host.cursor { + // The startup tail replay already folded this sequence. + return; + } + let deltas = host.roster.apply(&RosterEvent::from_record(&record)); + host.cursor = host.cursor.max(commit.sequence); + (deltas, host.cursor, host.roster.snapshot().to_string()) + }; + if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( + AGENT_ROSTER_REDUCER_ID, + AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ) { + eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); + } + // Socket echo events already committed their projection and change + // broadcast on the direct report path; their fold is roster-only. + let echo = ingress + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(SOCKET_REPORT_ADAPTER); + if echo { return; + } + for delta in deltas { + self.apply_roster_delta(delta, &ingress.kind); + } + } + + /// Apply one roster delta's side effects: the durable agent projection + /// commit and the agent-changed broadcast remote frontends converge on. + /// A removal commits the done state (history keeps the exit; the roster + /// already dropped the live entry). + fn apply_roster_delta(&self, delta: crate::journal_reducers::RosterDelta, kind: &str) { + use crate::journal_reducers::RosterDelta; + let (terminal_id, state, source, session) = match delta { + RosterDelta::Upsert { terminal_id, entry } => { + (terminal_id, entry.agent_state(), entry.agent_source(), entry.session) + } + RosterDelta::Remove { terminal_id } => { + (terminal_id, AgentState::Done, AgentSource::Hook, None) + } }; + let Ok(terminal_id) = TerminalPublicId::parse(&terminal_id) else { return }; let Some(surface) = self.resource_surface_for_terminal(&terminal_id) else { return }; - // The record's session field is a human-facing label; native agent - // session ids are opaque, so views fall back to their own context. - if let Err(error) = self.report_agent(surface, state, AgentSource::Hook, None) { + let mutation = match WorkspaceMutation::new( + format!("roster-{}", crate::workspace_registry::new_uuid_v4()), + "journal-reducer", + ) { + Ok(mutation) => mutation, + Err(_) => return, + }; + let fingerprint = serde_json::json!({ + "operation":"agent.report", + "surface":surface, + "state":state.as_str(), + "source":source.as_str(), + "source_session":session, + }); + if let Err(error) = self.commit_agent_report( + AgentReportTarget::Surface(surface), + state, + source, + session, + None, + &mutation, + &fingerprint, + AgentReportOrigin::RosterFold, + ) { eprintln!( - "cmux-tui: agent record update for {} ({}) failed: {error}", - terminal_id, ingress.kind + "cmux-tui: agent projection update for {terminal_id} ({kind}) failed: {error}" ); - return; } - // An ended session leaves the roster entirely: the done state was - // committed and broadcast above (so remote caches converge), and the - // live record is dropped so agents views stop listing the terminal - // and a fresh agent there starts clean. Hooks of one terminal are - // sequential (they follow one agent process's lifecycle), so nothing - // races this removal. - if state == AgentState::Done { - self.agent_records.lock().unwrap().remove(&terminal_id); + } + + /// Record a direct socket/SDK agent report in the journal so the roster + /// reducer (and any future reducer) sees every agent intent in one log. + /// The event wears the agent-hook payload shape with a dedicated + /// adapter, and the fold recognizes that adapter as an echo whose + /// projection commit already happened. + fn append_agent_report_echo( + &self, + terminal_id: &TerminalPublicId, + state: AgentState, + source: AgentSource, + session: Option<&str>, + updated_at_ms: u64, + ) { + use crate::journal_reducers::{SOCKET_REPORT_ADAPTER, SOCKET_REPORT_NATIVE_EVENT}; + let ingress = crate::JournalIngress { + producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), + manifest_version: crate::agent_hooks::AGENT_HOOK_MANIFEST_VERSION, + kind: "agent.state.changed".into(), + schema_version: 1, + occurred_at_ms: None, + subjects: vec![crate::JournalSubject { + kind: "terminal".into(), + id: terminal_id.to_string(), + }], + sensitivity: Some(crate::JournalSensitivity::Sensitive), + payload: serde_json::json!({ + "format": crate::agent_hooks::AGENT_HOOK_FORMAT, + "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, + "native_event": SOCKET_REPORT_NATIVE_EVENT, + "normalized": { + "state": state.as_str(), + "source": source.as_str(), + "source_session": session, + // The direct commit's timestamp, so the roster mirrors + // the projection exactly instead of stamping fold time. + "updated_at_ms": updated_at_ms.to_string(), + }, + "native": {}, + }), + causation_id: None, + correlation_id: None, + }; + let idempotency_key = + format!("agent-report-echo-{}", crate::workspace_registry::new_uuid_v4()); + if let Err(error) = self.append_journal_ingress(&ingress, "agent-report", &idempotency_key) + { + eprintln!("cmux-tui: journaling an agent report for {terminal_id} failed: {error}"); } } @@ -8590,6 +8772,7 @@ impl Mux { None, &mutation, &fingerprint, + AgentReportOrigin::Direct, )?; record.context("fresh raw agent report unexpectedly replayed") } @@ -8621,6 +8804,7 @@ impl Mux { expected_revision, mutation, &fingerprint, + AgentReportOrigin::Direct, ) .map(|(commit, _)| commit) } @@ -8635,6 +8819,7 @@ impl Mux { expected_revision: Option, mutation: &WorkspaceMutation, fingerprint: &Value, + origin: AgentReportOrigin, ) -> anyhow::Result<(ResourcePatchCommit, Option)> { let mut registry = self.workspace_registry.lock().unwrap(); if let Some(replay) = @@ -8676,14 +8861,27 @@ impl Mux { } }; let now = now_ms(); - let mut records = self.agent_records.lock().unwrap(); - let record = match records.get(&terminal_id) { - Some(existing) - if existing.source == AgentSource::Hook && source == AgentSource::Socket => - { - existing.clone() - } - _ => TerminalAgentRecord { + // Hook state is live agent truth: a direct socket report cannot + // overwrite it, so the commit re-asserts the existing hook values. + // Arbitration reads the durable projection under the registry lock + // held by this commit, which keeps concurrent reports serialized; + // the roster converges on the same outcome through the echo fold. + let existing = + registry.public_agent_projections(Some(&terminal_id), None)?.into_iter().next().filter( + |projection| { + projection.source == AgentSource::Hook.as_str() + && projection.state != AgentState::Done.as_str() + && source == AgentSource::Socket + }, + ); + let record = match existing { + Some(existing) => TerminalAgentRecord { + state: parse_projection_agent_state(&existing.state), + source: AgentSource::Hook, + session: existing.source_session, + updated_at_ms: existing.updated_at_ms, + }, + None => TerminalAgentRecord { state: agent_state, source, session: source_session, @@ -8720,10 +8918,6 @@ impl Mux { &deltas, )?; state.resource_revision = commit.revision; - if !commit.replayed { - records.insert(terminal_id.clone(), record.clone()); - } - drop(records); drop(state); drop(registry); let agent = AgentRecord { @@ -8743,6 +8937,18 @@ impl Mux { session: agent.session.as_deref().map(Arc::from), updated_at_ms: agent.updated_at_ms, }); + if origin == AgentReportOrigin::Direct { + // The roster only folds journal events, so a direct report + // records its intent in the log; the fold recognizes the + // echo adapter and applies it roster-only. + self.append_agent_report_echo( + &agent.terminal_id, + agent.state, + agent.source, + agent.session.as_deref(), + agent.updated_at_ms, + ); + } } Ok((commit, Some(agent))) } @@ -8764,7 +8970,26 @@ impl Mux { } fn purge_terminal_side_tables(&self, terminal_id: &TerminalPublicId) { - self.agent_records.lock().unwrap().remove(terminal_id); + // Terminal lifecycle does not flow through `agent.*` journal events + // yet, so a closed terminal retires its roster entry explicitly. + // The snapshot persists so a restart does not resurrect the entry; + // the roster lock is released before the registry lock per the + // host's lock-ordering rule. + let retired = { + let mut host = self.agent_roster.lock().unwrap(); + let retired = host.roster.retire_terminal(terminal_id.as_str()); + retired.then(|| (host.cursor, host.roster.snapshot().to_string())) + }; + if let Some((cursor, snapshot)) = retired + && let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ) + { + eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); + } self.terminal_notifications.lock().unwrap().remove(terminal_id); } @@ -8792,7 +9017,7 @@ impl Mux { surface: Option, state: Option, ) -> Vec { - let records = self.agent_records.lock().unwrap().clone(); + let entries = self.agent_roster.lock().unwrap().roster.entries.clone(); let state_snapshot = self.state.lock().unwrap(); let requested_terminal = surface.and_then(|surface| { state_snapshot @@ -8801,9 +9026,10 @@ impl Mux { .or_else(|| state_snapshot.terminal_runtime_by_id(surface)) .and_then(|surface| surface.terminal_public_id().cloned()) }); - let mut records = records + let mut records = entries .into_iter() - .filter_map(|(terminal_id, record)| { + .filter_map(|(terminal_id, entry)| { + let terminal_id = TerminalPublicId::parse(terminal_id).ok()?; let representative = state_snapshot .placements_of_content(&ContentPublicId::Terminal(terminal_id.clone())) .first() @@ -8814,10 +9040,10 @@ impl Mux { Some(AgentRecord { surface: representative, terminal_id, - state: record.state, - source: record.source, - session: record.session, - updated_at_ms: record.updated_at_ms, + state: entry.agent_state(), + source: entry.agent_source(), + session: entry.session, + updated_at_ms: entry.updated_at_ms, }) }) .collect::>(); @@ -21322,7 +21548,9 @@ mod tests { assert_eq!(filtered[0].session.as_deref(), Some("hook-session")); assert!(mux.list_agents(Some(surface.id), Some(AgentState::Done)).is_empty()); assert_eq!(mux.with_state(|state| state.resource_revision), initial_revision + 3); - assert_eq!(mux.resource_event_epoch(), initial_epoch + 3); + // Each fresh direct report publishes twice on the shared change + // epoch: its resource commit and its journal echo. + assert_eq!(mux.resource_event_epoch(), initial_epoch + 6); assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 1); let resource_events = mux.resource_events_after(initial_revision).unwrap(); assert_eq!(resource_events.batches.len(), 3); @@ -21426,7 +21654,9 @@ mod tests { assert_eq!(ignored.source, AgentSource::Hook); assert_eq!(ignored.session.as_deref(), Some("hook-session")); assert_eq!(mux.with_state(|state| state.resource_revision), created_revision + 3); - assert_eq!(mux.resource_event_epoch(), initial_epoch + 3); + // Each fresh direct report publishes twice on the shared change + // epoch: its resource commit and its journal echo. + assert_eq!(mux.resource_event_epoch(), initial_epoch + 6); assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 1); let batches = mux.resource_events_after(created_revision).unwrap().batches; @@ -21658,6 +21888,124 @@ mod tests { assert!(mux.list_agents(None, None).is_empty()); } + #[test] + fn raw_socket_report_reaches_the_roster_through_its_journal_echo() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("probe".into()), + ) + .unwrap(); + let records = mux.list_agents(Some(surface.id), None); + assert_eq!(records.len(), 1); + assert_eq!(records[0].state, AgentState::Working); + assert_eq!(records[0].source, AgentSource::Socket); + assert_eq!(records[0].session.as_deref(), Some("probe")); + // The roster only folds journal events, so the record's presence + // proves the direct report echoed its intent into the journal. + let echoes = mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 512) + .unwrap() + .records + .into_iter() + .filter(|record| record.kind == "agent.state.changed") + .count(); + assert_eq!(echoes, 1); + } + + #[test] + fn agent_roster_rederives_from_the_journal_head_without_its_snapshot() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-rederive-{}", crate::workspace_registry::new_uuid_v4())); + let session = "roster-rederive"; + let (terminal_id, live_entries) = { + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + let mux = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = mux.with_state(|state| { + match state.resource_indexes.content_ids.get(&surface.id).unwrap() { + ContentPublicId::Terminal(terminal_id) => terminal_id.clone(), + ContentPublicId::Browser(_) => panic!("workspace opened a browser"), + } + }); + let append = |event: &str, key: &str| { + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + event, + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"native-1"}), + ) + .unwrap(); + mux.append_journal_ingress(&ingress, "test", key).unwrap(); + }; + append("SessionStart", "rederive-1"); + append("UserPromptSubmit", "rederive-2"); + let entries = mux.agent_roster.lock().unwrap().roster.entries.clone(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[terminal_id.as_str()].state, "working"); + mux.shutdown(); + (terminal_id, entries) + }; + + // Wipe the persisted reducer state so the reopen cannot lean on the + // snapshot: an identical roster proves it derives from the journal. + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + registry + .put_journal_reducer_state(crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, 0, 0, "") + .unwrap(); + let reopened = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let rederived = reopened.agent_roster.lock().unwrap().roster.entries.clone(); + assert_eq!(rederived, live_entries); + + // Folding the tail after an ended session removes the entry, and + // that removal survives the next reopen through the snapshot. + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "SessionEnd", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"native-1"}), + ) + .unwrap(); + reopened.append_journal_ingress(&ingress, "test", "rederive-3").unwrap(); + assert!(reopened.agent_roster.lock().unwrap().roster.entries.is_empty()); + reopened.shutdown(); + drop(reopened); + + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + let final_mux = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + assert!(final_mux.agent_roster.lock().unwrap().roster.entries.is_empty()); + final_mux.shutdown(); + drop(final_mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn failed_raw_agent_report_rolls_back_projection_memory_revision_and_event() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs index 965968c0c1b8..2dbdd4365d51 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs @@ -8,7 +8,6 @@ pub(super) struct RestoredPublicProjections { pub(super) default_colors: DefaultColors, pub(super) has_terminal_defaults: bool, pub(super) next_notification_id: u64, - pub(super) agent_records: HashMap, pub(super) terminal_notifications: HashMap, pub(super) notification_ledger: VecDeque, } @@ -58,29 +57,12 @@ pub(super) fn restore_public_projections( .context("notification count exceeds uint64")? .saturating_add(1); - let mut agent_records = HashMap::with_capacity(projections.agents.len()); - for agent in projections.agents { - let previous = agent_records.insert( - agent.terminal_id.clone(), - TerminalAgentRecord { - state: agent_state(&agent.state)?, - source: agent_source(&agent.source)?, - session: agent.source_session, - updated_at_ms: agent.updated_at_ms, - }, - ); - anyhow::ensure!( - previous.is_none(), - "multiple durable agents resolve to terminal {}", - agent.terminal_id - ); - } - + // The live agent roster derives from the journal (snapshot plus tail + // fold), not from the durable agent projections restored here. Ok(RestoredPublicProjections { default_colors, has_terminal_defaults, next_notification_id, - agent_records, terminal_notifications, notification_ledger, }) @@ -95,26 +77,6 @@ fn notification_level(value: &str) -> anyhow::Result { } } -fn agent_state(value: &str) -> anyhow::Result { - match value { - "working" => Ok(AgentState::Working), - "blocked" => Ok(AgentState::Blocked), - "idle" => Ok(AgentState::Idle), - "done" => Ok(AgentState::Done), - "unknown" => Ok(AgentState::Unknown), - other => anyhow::bail!("invalid durable agent state {other:?}"), - } -} - -fn agent_source(value: &str) -> anyhow::Result { - match value { - "detected" => Ok(AgentSource::Detected), - "socket" => Ok(AgentSource::Socket), - "hook" => Ok(AgentSource::Hook), - other => anyhow::bail!("invalid durable agent source {other:?}"), - } -} - #[cfg(test)] mod tests { use super::*; @@ -185,7 +147,6 @@ mod tests { state.terminal_catalog_by_runtime.insert(runtime.terminal_runtime_id().unwrap(), terminal); let restored = restore_public_projections(&state, projections).unwrap(); let terminal = TerminalPublicId::parse("term_00000000000000000000000000000001").unwrap(); - assert_eq!(restored.agent_records.get(&terminal).unwrap().state, AgentState::Working); assert_eq!(restored.notification_ledger[0].terminal_id.as_ref(), Some(&terminal)); assert_eq!(restored.notification_ledger[0].surface, Some(runtime.id)); assert!(restored.terminal_notifications[&terminal].unread); diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index b9041687c680..e9f8f6352276 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -18900,7 +18900,9 @@ mod tests { assert_eq!(result["source"], "socket"); assert_eq!(result["session"], "raw-command"); assert_eq!(mux.with_state(|state| state.resource_revision), revision + 1); - assert_eq!(mux.resource_event_epoch(), epoch + 1); + // A fresh direct report publishes twice on the shared change epoch: + // its resource commit and its journal echo. + assert_eq!(mux.resource_event_epoch(), epoch + 2); assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 1); let events = mux.resource_events_after(revision).unwrap(); assert_eq!(events.batches.len(), 1); diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs index 094a425c3682..4ea6c9672672 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs @@ -886,6 +886,57 @@ impl WorkspaceRegistry { query_session_journal_after(&self.connection, sequence, limit) } + /// Persisted fold position of one journal reducer: (version, cursor, + /// snapshot). A version mismatch on load discards the snapshot so the + /// reducer re-folds from the journal head. + pub(crate) fn journal_reducer_state( + &self, + reducer_id: &str, + ) -> anyhow::Result> { + let raw = self + .connection + .query_row( + "SELECT value FROM meta WHERE key = ?1", + [format!("journal_reducer.{reducer_id}")], + |row| row.get::<_, String>(0), + ) + .optional()?; + let Some(raw) = raw else { return Ok(None) }; + let value: Value = serde_json::from_str(&raw) + .with_context(|| format!("journal reducer state for {reducer_id} is not JSON"))?; + let version = value.get("version").and_then(Value::as_u64).unwrap_or(0) as u32; + let cursor = value + .get("cursor") + .and_then(Value::as_str) + .and_then(|cursor| cursor.parse::().ok()) + .unwrap_or(0); + let snapshot = + value.get("snapshot").and_then(Value::as_str).map(str::to_string).unwrap_or_default(); + Ok(Some((version, cursor, snapshot))) + } + + /// Durably record a reducer's fold position and state snapshot. Cursor + /// values are stored as strings so 64-bit sequences survive JSON. + pub(crate) fn put_journal_reducer_state( + &self, + reducer_id: &str, + version: u32, + cursor: u64, + snapshot: &str, + ) -> anyhow::Result<()> { + let value = serde_json::json!({ + "version": version, + "cursor": cursor.to_string(), + "snapshot": snapshot, + }); + self.connection.execute( + "INSERT INTO meta(key, value) VALUES(?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![format!("journal_reducer.{reducer_id}"), value.to_string()], + )?; + Ok(()) + } + /// The most recently started journal output stream for one terminal: /// its generation and the exclusive end offset of its journaled bytes. pub(crate) fn terminal_stream_latest( From 8da5643df4d89ecf4b3a0abad5809241c57e6b1d Mon Sep 17 00:00:00 2001 From: lawrencecchen Date: Thu, 27 Aug 2026 20:45:22 -0700 Subject: [PATCH 005/133] feat(tui): carry the agent adapter id through the roster to views Every hook journal event names its adapter (claude, codex, ...); the roster keeps it, agent records / the agent-changed event / AgentInfo expose it, and views can label rows by agent type instead of a short id. Socket-only reports leave it absent until a hook claims the terminal. Reducer snapshot version bumps to 2. The durable projection JSON is deliberately unchanged this round (SDK-facing schema; its equality contracts stay intact) - agent type there is a follow-up. --- .../crates/cmux-tui-core/src/event_bus.rs | 4 ++ .../cmux-tui-core/src/journal_reducers.rs | 62 ++++++++++++------- cmux-tui/crates/cmux-tui-core/src/mux.rs | 25 ++++++-- cmux-tui/crates/cmux-tui-core/src/server.rs | 6 +- cmux-tui/crates/cmux-tui/src/app.rs | 2 + cmux-tui/crates/cmux-tui/src/session/mod.rs | 4 ++ .../crates/cmux-tui/src/session/remote.rs | 5 ++ .../crates/cmux-tui/src/sidebar_projection.rs | 1 + 8 files changed, 79 insertions(+), 30 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/event_bus.rs b/cmux-tui/crates/cmux-tui-core/src/event_bus.rs index b1bfccb2abe2..563fd81432a6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/event_bus.rs +++ b/cmux-tui/crates/cmux-tui-core/src/event_bus.rs @@ -466,6 +466,7 @@ mod tests { state: format!("one-{index}").into(), source: "hook".into(), session: None, + agent: None, updated_at_ms: index, }); broadcaster.emit(MuxEvent::AgentChanged { @@ -473,6 +474,7 @@ mod tests { state: format!("two-{index}").into(), source: "socket".into(), session: Some("agent-session".into()), + agent: None, updated_at_ms: index, }); } @@ -493,6 +495,7 @@ mod tests { state, source, session: Some(session), + agent: None, updated_at_ms: 9_999, } if state.as_ref() == "two-9999" && source.as_ref() == "socket" @@ -558,6 +561,7 @@ mod tests { state: "working".into(), source: "hook".into(), session: None, + agent: None, updated_at_ms: 1, }); broadcaster.emit(MuxEvent::SurfaceExited(4)); diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index a59eb54cc66a..4aa46372d093 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -27,7 +27,8 @@ use crate::{AgentSource, AgentState, JournalSubject}; pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; /// Bump to discard persisted snapshots and re-fold from the journal head. -pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 1; +/// Version 2 added the agent adapter id to roster entries. +pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 2; /// The adapter id and native event the socket report path uses for its echo /// journal events. The echo carries the explicit state in `normalized`, so @@ -116,6 +117,11 @@ pub(crate) struct RosterEntry { pub(crate) state: String, pub(crate) source: String, pub(crate) session: Option, + /// The reporting adapter id (`claude`, `codex`, ...). Direct socket + /// reports do not know the agent behind the terminal, so it is absent + /// there until a hook event claims the terminal. + #[serde(default)] + pub(crate) agent: Option, pub(crate) updated_at_ms: u64, } @@ -157,28 +163,29 @@ impl AgentRoster { return Vec::new(); } let Some(terminal_id) = event.terminal_id() else { return Vec::new() }; - let (state, source, session, updated_at_ms) = if event.adapter_id() - == Some(SOCKET_REPORT_ADAPTER) - { - // Socket echo: explicit state and timestamp carried in the - // payload, so the roster mirrors the direct projection - // commit exactly. - let Some(state) = event.normalized("state").and_then(agent_state_from_str) else { - return Vec::new(); + let (state, source, session, agent, updated_at_ms) = + if event.adapter_id() == Some(SOCKET_REPORT_ADAPTER) { + // Socket echo: explicit state and timestamp carried in the + // payload, so the roster mirrors the direct projection + // commit exactly. The reporter does not know the agent type. + let Some(state) = event.normalized("state").and_then(agent_state_from_str) else { + return Vec::new(); + }; + let source = event + .normalized("source") + .and_then(agent_source_from_str) + .unwrap_or(AgentSource::Socket); + let updated_at_ms = event + .normalized("updated_at_ms") + .and_then(|value| value.parse::().ok()) + .unwrap_or(event.committed_at_ms); + let session = event.normalized("source_session").map(str::to_string); + (state, source, session, None, updated_at_ms) + } else { + let Some(state) = state_for_hook_kind(event.kind) else { return Vec::new() }; + let agent = event.adapter_id().map(str::to_string); + (state, AgentSource::Hook, None, agent, event.committed_at_ms) }; - let source = event - .normalized("source") - .and_then(agent_source_from_str) - .unwrap_or(AgentSource::Socket); - let updated_at_ms = event - .normalized("updated_at_ms") - .and_then(|value| value.parse::().ok()) - .unwrap_or(event.committed_at_ms); - (state, source, event.normalized("source_session").map(str::to_string), updated_at_ms) - } else { - let Some(state) = state_for_hook_kind(event.kind) else { return Vec::new() }; - (state, AgentSource::Hook, None, event.committed_at_ms) - }; if source == AgentSource::Socket && self .entries @@ -203,6 +210,10 @@ impl AgentRoster { state: state.as_str().to_string(), source: source.as_str().to_string(), session, + // A socket entry keeps any agent identity a hook already + // established for this terminal. + agent: agent + .or_else(|| self.entries.get(terminal_id).and_then(|entry| entry.agent.clone())), updated_at_ms, }; if self.entries.get(terminal_id) == Some(&entry) { @@ -260,6 +271,7 @@ mod tests { roster.apply(&hook_event(1, "agent.session.started", &subjects, &payload)); assert_eq!(roster.entries["term_a"].state, "idle"); + assert_eq!(roster.entries["term_a"].agent, None, "payload without adapter has no agent"); roster.apply(&hook_event(2, "agent.turn.started", &subjects, &payload)); assert_eq!(roster.entries["term_a"].state, "working"); @@ -291,11 +303,13 @@ mod tests { assert_eq!(entry.state, "working"); assert_eq!(entry.source, "socket"); assert_eq!(entry.session.as_deref(), Some("probe")); + assert_eq!(entry.agent, None); - // A hook event takes the terminal over... - let hook_payload = json!({}); + // A hook event takes the terminal over and names the agent... + let hook_payload = json!({"adapter": {"id": "claude", "version": 1}}); roster.apply(&hook_event(2, "agent.turn.started", &subjects, &hook_payload)); assert_eq!(roster.entries["term_a"].source, "hook"); + assert_eq!(roster.entries["term_a"].agent.as_deref(), Some("claude")); // ...and later socket reports cannot downgrade it. let deltas = diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5fecb685e62d..077fd9981380 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -854,6 +854,7 @@ pub enum MuxEvent { state: Arc, source: Arc, session: Option>, + agent: Option>, updated_at_ms: u64, }, Bell(SurfaceId), @@ -1154,6 +1155,9 @@ pub struct AgentRecord { pub state: AgentState, pub source: AgentSource, pub session: Option, + /// The reporting adapter id (`claude`, `codex`, ...) when a hook has + /// claimed the terminal; absent for socket-only reports. + pub agent: Option, pub updated_at_ms: u64, } @@ -5323,12 +5327,16 @@ impl Mux { /// already dropped the live entry). fn apply_roster_delta(&self, delta: crate::journal_reducers::RosterDelta, kind: &str) { use crate::journal_reducers::RosterDelta; - let (terminal_id, state, source, session) = match delta { - RosterDelta::Upsert { terminal_id, entry } => { - (terminal_id, entry.agent_state(), entry.agent_source(), entry.session) - } + let (terminal_id, state, source, session, agent_adapter) = match delta { + RosterDelta::Upsert { terminal_id, entry } => ( + terminal_id, + entry.agent_state(), + entry.agent_source(), + entry.session.clone(), + entry.agent, + ), RosterDelta::Remove { terminal_id } => { - (terminal_id, AgentState::Done, AgentSource::Hook, None) + (terminal_id, AgentState::Done, AgentSource::Hook, None, None) } }; let Ok(terminal_id) = TerminalPublicId::parse(&terminal_id) else { return }; @@ -5356,6 +5364,7 @@ impl Mux { &mutation, &fingerprint, AgentReportOrigin::RosterFold, + agent_adapter, ) { eprintln!( "cmux-tui: agent projection update for {terminal_id} ({kind}) failed: {error}" @@ -8773,6 +8782,7 @@ impl Mux { &mutation, &fingerprint, AgentReportOrigin::Direct, + None, )?; record.context("fresh raw agent report unexpectedly replayed") } @@ -8805,6 +8815,7 @@ impl Mux { mutation, &fingerprint, AgentReportOrigin::Direct, + None, ) .map(|(commit, _)| commit) } @@ -8820,6 +8831,7 @@ impl Mux { mutation: &WorkspaceMutation, fingerprint: &Value, origin: AgentReportOrigin, + agent_adapter: Option, ) -> anyhow::Result<(ResourcePatchCommit, Option)> { let mut registry = self.workspace_registry.lock().unwrap(); if let Some(replay) = @@ -8926,6 +8938,7 @@ impl Mux { state: record.state, source: record.source, session: record.session, + agent: agent_adapter, updated_at_ms: record.updated_at_ms, }; if !commit.replayed { @@ -8935,6 +8948,7 @@ impl Mux { state: Arc::from(agent.state.as_str()), source: Arc::from(agent.source.as_str()), session: agent.session.as_deref().map(Arc::from), + agent: agent.agent.as_deref().map(Arc::from), updated_at_ms: agent.updated_at_ms, }); if origin == AgentReportOrigin::Direct { @@ -9043,6 +9057,7 @@ impl Mux { state: entry.agent_state(), source: entry.agent_source(), session: entry.session, + agent: entry.agent, updated_at_ms: entry.updated_at_ms, }) }) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index e9f8f6352276..344b70ce3b56 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -10081,6 +10081,7 @@ fn agent_json(record: &AgentRecord) -> Value { "state": record.state.as_str(), "source": record.source.as_str(), "session": record.session, + "agent": record.agent, "updated_at_ms": record.updated_at_ms, }) } @@ -13012,12 +13013,13 @@ fn subscribed_event_json(event: &MuxEvent) -> Value { MuxEvent::TitleChanged { surface, title } => { json!({"event": "title-changed", "surface": surface, "title": title.as_ref()}) } - MuxEvent::AgentChanged { surface, state, source, session, updated_at_ms } => json!({ + MuxEvent::AgentChanged { surface, state, source, session, agent, updated_at_ms } => json!({ "event": "agent-changed", "surface": surface, "state": state.as_ref(), "source": source.as_ref(), "session": session.as_deref(), + "agent": agent.as_deref(), "updated_at_ms": updated_at_ms, }), MuxEvent::Bell(id) => json!({"event": "bell", "surface": id}), @@ -22394,6 +22396,7 @@ mod tests { state: Arc::::from("working"), source: Arc::::from("hook"), session: Some(Arc::::from("review")), + agent: Some(Arc::::from("claude")), updated_at_ms: 41, }), json!({ @@ -22402,6 +22405,7 @@ mod tests { "state": "working", "source": "hook", "session": "review", + "agent": "claude", "updated_at_ms": 41, }) ); diff --git a/cmux-tui/crates/cmux-tui/src/app.rs b/cmux-tui/crates/cmux-tui/src/app.rs index f75b40be2ce6..51e19abe1db9 100644 --- a/cmux-tui/crates/cmux-tui/src/app.rs +++ b/cmux-tui/crates/cmux-tui/src/app.rs @@ -32830,6 +32830,7 @@ mod tests { state: "working".into(), source: "hook".into(), session: None, + agent: None, updated_at_ms: 1, }, &tx, @@ -32852,6 +32853,7 @@ mod tests { state: "working".into(), source: "hook".into(), session: None, + agent: None, updated_at_ms: 2, }, &tx, diff --git a/cmux-tui/crates/cmux-tui/src/session/mod.rs b/cmux-tui/crates/cmux-tui/src/session/mod.rs index cc5de5529048..4689b3210fa8 100644 --- a/cmux-tui/crates/cmux-tui/src/session/mod.rs +++ b/cmux-tui/crates/cmux-tui/src/session/mod.rs @@ -308,6 +308,9 @@ pub struct AgentInfo { pub state: String, pub source: String, pub session: Option, + /// The reporting adapter id (`claude`, `codex`, ...), when known. + #[serde(default)] + pub agent: Option, pub updated_at_ms: u64, } @@ -930,6 +933,7 @@ impl Session { state: agent.state.as_str().to_string(), source: agent.source.as_str().to_string(), session: agent.session, + agent: agent.agent, updated_at_ms: agent.updated_at_ms, }) .collect(), diff --git a/cmux-tui/crates/cmux-tui/src/session/remote.rs b/cmux-tui/crates/cmux-tui/src/session/remote.rs index d89270f847dd..bcc1815cf83b 100644 --- a/cmux-tui/crates/cmux-tui/src/session/remote.rs +++ b/cmux-tui/crates/cmux-tui/src/session/remote.rs @@ -2324,11 +2324,13 @@ impl RemoteSession { return; }; let session = value.get("session").and_then(Value::as_str).map(str::to_string); + let agent_adapter = value.get("agent").and_then(Value::as_str).map(str::to_string); let agent = AgentInfo { surface, state: state.to_string(), source: source.to_string(), session, + agent: agent_adapter, updated_at_ms, }; let event = MuxEvent::AgentChanged { @@ -2336,6 +2338,7 @@ impl RemoteSession { state: Arc::from(agent.state.as_str()), source: Arc::from(agent.source.as_str()), session: agent.session.as_deref().map(Arc::from), + agent: agent.agent.as_deref().map(Arc::from), updated_at_ms, }; self.tree.lock().unwrap().update_agent(agent); @@ -7337,6 +7340,7 @@ mod tests { state: "blocked".into(), source: "hook".into(), session: Some("review".into()), + agent: None, updated_at_ms: 41, }] ); @@ -7606,6 +7610,7 @@ mod tests { state: "working".into(), source: "hook".into(), session: Some("review".into()), + agent: None, updated_at_ms: 41, }); diff --git a/cmux-tui/crates/cmux-tui/src/sidebar_projection.rs b/cmux-tui/crates/cmux-tui/src/sidebar_projection.rs index 1072135e3a7e..3c7b2955fddf 100644 --- a/cmux-tui/crates/cmux-tui/src/sidebar_projection.rs +++ b/cmux-tui/crates/cmux-tui/src/sidebar_projection.rs @@ -356,6 +356,7 @@ mod tests { state: "working".into(), source: "detected".into(), session: Some("fix sidebar".into()), + agent: None, updated_at_ms: 1, }]; let rows = rows( From d8d7abe87d0198efc7098c224b24f3028821eda2 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:16:02 -0700 Subject: [PATCH 006/133] fix(tui): force socket roster source precedence --- .../cmux-tui-core/src/journal_reducers.rs | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 4aa46372d093..f1b67a5584a3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -171,10 +171,10 @@ impl AgentRoster { let Some(state) = event.normalized("state").and_then(agent_state_from_str) else { return Vec::new(); }; - let source = event - .normalized("source") - .and_then(agent_source_from_str) - .unwrap_or(AgentSource::Socket); + // The socket adapter is authoritative about origin. Do not + // trust the user-controlled normalized source field, or it + // could bypass hook-over-socket precedence. + let source = AgentSource::Socket; let updated_at_ms = event .normalized("updated_at_ms") .and_then(|value| value.parse::().ok()) @@ -318,6 +318,25 @@ mod tests { assert_eq!(roster.entries["term_a"].source, "hook"); } + #[test] + fn socket_echo_cannot_spoof_hook_source() { + let subjects = terminal_subject("term_a"); + let hook_payload = json!({"adapter": {"id": "claude", "version": 1}}); + let socket_payload = json!({ + "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, + "normalized": {"state": "idle", "source": "hook"} + }); + let mut roster = AgentRoster::default(); + roster.apply(&hook_event(1, "agent.turn.started", &subjects, &hook_payload)); + assert!( + roster + .apply(&hook_event(2, "agent.state.changed", &subjects, &socket_payload)) + .is_empty() + ); + assert_eq!(roster.entries["term_a"].state, "working"); + assert_eq!(roster.entries["term_a"].source, "hook"); + } + #[test] fn folds_are_idempotent_per_state_and_deterministic_across_replays() { let subjects = terminal_subject("term_a"); From aefa61018540642615074983ca6e4d4c8fba133d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:14:07 -0700 Subject: [PATCH 007/133] fix(tui): keep reducer agent identity semantics stable --- .../crates/cmux-tui-core/src/journal_reducers.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index f1b67a5584a3..74ad270268b1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -212,11 +212,19 @@ impl AgentRoster { session, // A socket entry keeps any agent identity a hook already // established for this terminal. - agent: agent - .or_else(|| self.entries.get(terminal_id).and_then(|entry| entry.agent.clone())), + agent: if source == AgentSource::Socket { + agent.or_else(|| self.entries.get(terminal_id).and_then(|entry| entry.agent.clone())) + } else { + agent + }, updated_at_ms, }; - if self.entries.get(terminal_id) == Some(&entry) { + if self.entries.get(terminal_id).is_some_and(|existing| { + existing.state == entry.state + && existing.source == entry.source + && existing.session == entry.session + && existing.agent == entry.agent + }) { return Vec::new(); } self.entries.insert(terminal_id.to_string(), entry.clone()); From a21ed151fc93242b75eed0e5e4c667bff15f147d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:23:05 -0700 Subject: [PATCH 008/133] fix(tui): carry optional agent identity through records --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8c4c4dce60f7..f3317bdd1c46 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -854,6 +854,7 @@ pub enum MuxEvent { state: Arc, source: Arc, session: Option>, + agent: Option>, updated_at_ms: u64, }, Bell(SurfaceId), @@ -1111,6 +1112,7 @@ pub struct AgentRecord { pub state: AgentState, pub source: AgentSource, pub session: Option, + pub agent: Option, pub updated_at_ms: u64, } @@ -1183,6 +1185,7 @@ struct TerminalAgentRecord { state: AgentState, source: AgentSource, session: Option, + agent: Option, updated_at_ms: u64, } @@ -9176,6 +9179,7 @@ impl Mux { state: agent_state, source, session: source_session, + agent: None, updated_at_ms: now, }, }; @@ -9264,6 +9268,7 @@ impl Mux { state: record.state, source: record.source, session: record.session, + agent: record.agent, updated_at_ms: record.updated_at_ms, }; if !commit.replayed { @@ -9273,6 +9278,7 @@ impl Mux { state: Arc::from(agent.state.as_str()), source: Arc::from(agent.source.as_str()), session: agent.session.as_deref().map(Arc::from), + agent: agent.agent.as_deref().map(Arc::from), updated_at_ms: agent.updated_at_ms, }); } @@ -9358,6 +9364,7 @@ impl Mux { state: record.state, source: record.source, session: record.session, + agent: record.agent, updated_at_ms: record.updated_at_ms, }) }) From 377291db1d34676ac0b719699c6c6a2ef144b3fd Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 22:19:51 -0700 Subject: [PATCH 009/133] fix(tui): restore optional agent identity --- cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs index 8b222d314f26..7ad08f24e203 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs @@ -110,6 +110,7 @@ pub(super) fn restore_public_projections( state, source: agent_source(&agent.source)?, session: (!internal_marker).then_some(agent.source_session).flatten(), + agent: None, updated_at_ms: agent.updated_at_ms, }, ); From c9ad5585ce06bddefc35078d57ccab32f7953a67 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 22:22:59 -0700 Subject: [PATCH 010/133] style(tui): apply hosted cargo fmt --- cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 74ad270268b1..b3cfad2990b5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -213,7 +213,8 @@ impl AgentRoster { // A socket entry keeps any agent identity a hook already // established for this terminal. agent: if source == AgentSource::Socket { - agent.or_else(|| self.entries.get(terminal_id).and_then(|entry| entry.agent.clone())) + agent + .or_else(|| self.entries.get(terminal_id).and_then(|entry| entry.agent.clone())) } else { agent }, From 869b19c7f93f9a5e9c52d4783ff887247b4d0008 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:33:50 -0700 Subject: [PATCH 011/133] test(tui): require durable agent roster lifecycle --- .../cmux-tui-core/src/journal_reducers.rs | 13 +++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 80 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index b3cfad2990b5..c194567417e3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -374,6 +374,19 @@ mod tests { assert_eq!(serde_json::to_value(&live).unwrap(), serde_json::to_value(&replayed).unwrap()); } + #[test] + fn same_state_event_with_new_commit_time_refreshes_roster() { + let subjects = terminal_subject("term_a"); + let payload = json!({}); + let mut roster = AgentRoster::default(); + + roster.apply(&hook_event(1, "agent.turn.started", &subjects, &payload)); + let deltas = roster.apply(&hook_event(2, "agent.turn.started", &subjects, &payload)); + + assert_eq!(deltas.len(), 1); + assert_eq!(roster.entries["term_a"].updated_at_ms, 1_002); + } + #[test] fn snapshot_round_trips_and_foreign_producers_are_ignored() { let subjects = terminal_subject("term_a"); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f3317bdd1c46..bb0dc7e1bc46 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22164,6 +22164,86 @@ mod tests { assert_eq!(records[0].state, AgentState::Idle); } + #[test] + fn committed_hook_event_persists_roster_identity() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"native-1"}), + ) + .unwrap(); + + let commit = mux.append_journal_ingress(&ingress, "test", "roster-identity").unwrap(); + let records = mux.list_agents(Some(surface.id), None); + assert_eq!(records.len(), 1); + assert_eq!(records[0].agent.as_deref(), Some("claude")); + + let (version, cursor, snapshot) = mux + .workspace_registry + .lock() + .unwrap() + .journal_reducer_state(crate::journal_reducers::AGENT_ROSTER_REDUCER_ID) + .unwrap() + .expect("the committed event has a persisted roster fold"); + assert_eq!(version, crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION); + assert_eq!(cursor, commit.sequence); + let snapshot: Value = serde_json::from_str(&snapshot).unwrap(); + assert_eq!(snapshot["entries"][terminal_id.as_str()]["agent"], "claude"); + } + + #[test] + fn roster_identity_restores_from_the_journal_snapshot() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-restore-{}", crate::workspace_registry::new_uuid_v4())); + let session = "roster-restore"; + let terminal_id = { + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + let mux = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"native-1"}), + ) + .unwrap(); + mux.append_journal_ingress(&ingress, "test", "roster-restore").unwrap(); + assert_eq!(mux.list_agents(Some(surface.id), None)[0].agent.as_deref(), Some("claude")); + mux.shutdown(); + drop(mux); + terminal_id + }; + + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + let reopened = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let surface = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); + let records = reopened.list_agents(Some(surface.id), None); + assert_eq!(records.len(), 1); + assert_eq!(records[0].agent.as_deref(), Some("claude")); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn stale_same_terminal_hook_sequence_cannot_overwrite_newer_state() { let mux = test_mux(); From cef6b90a1fd3c7df73b806c50a323c82198c05b8 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:42:02 -0700 Subject: [PATCH 012/133] fix(tui): connect agent roster to journal lifecycle --- .../cmux-tui-core/src/journal_reducers.rs | 1 + cmux-tui/crates/cmux-tui-core/src/mux.rs | 283 +++++++++++++++++- 2 files changed, 276 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index c194567417e3..8ca3a6c3f2c5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -225,6 +225,7 @@ impl AgentRoster { && existing.source == entry.source && existing.session == entry.session && existing.agent == entry.agent + && existing.updated_at_ms == entry.updated_at_ms }) { return Vec::new(); } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bb0dc7e1bc46..b8f6b76035bf 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1087,6 +1087,48 @@ impl AgentSource { } } +/// The journal-derived agent roster and the last journal sequence folded into +/// it. The snapshot is durable; the cursor makes live folds and restart +/// replay idempotent. +#[derive(Debug, Default)] +struct AgentRosterHost { + roster: crate::journal_reducers::AgentRoster, + cursor: u64, +} + +fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { + use crate::journal_reducers::{ + AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, AgentRoster, RosterEvent, + }; + + let mut host = match registry.journal_reducer_state(AGENT_ROSTER_REDUCER_ID)? { + Some((version, cursor, snapshot)) if version == AGENT_ROSTER_REDUCER_VERSION => { + AgentRosterHost { roster: AgentRoster::restore(&snapshot).unwrap_or_default(), cursor } + } + _ => AgentRosterHost::default(), + }; + let initial_cursor = host.cursor; + loop { + let page = registry.session_journal_after(host.cursor, 512)?; + if page.records.is_empty() { + break; + } + for record in &page.records { + host.roster.apply(&RosterEvent::from_record(record)); + host.cursor = host.cursor.max(record.sequence); + } + } + if host.cursor != initial_cursor { + registry.put_journal_reducer_state( + AGENT_ROSTER_REDUCER_ID, + AGENT_ROSTER_REDUCER_VERSION, + host.cursor, + &host.roster.snapshot().to_string(), + )?; + } + Ok(host) +} + /// The agent-record state a committed hook journal event implies, or `None` /// for events that carry no lifecycle transition (child agents, unclassified /// state changes) so they never churn the record. @@ -1189,6 +1231,12 @@ struct TerminalAgentRecord { updated_at_ms: u64, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AgentReportOrigin { + Direct, + RosterFold, +} + enum AgentReportTarget<'a> { Surface(SurfaceId), Resource { selectors: &'a crate::ResourceSelectors, terminal_id: &'a TerminalPublicId }, @@ -2091,6 +2139,7 @@ pub struct Mux { default_colors: Mutex, durable_terminal_defaults: AtomicBool, sidebar_plugin: Mutex, + agent_roster: Mutex, agent_records: Mutex>, agent_hook_fences: Mutex>, /// Nonterminal notifications remain placement-local. Terminal unread @@ -2365,6 +2414,22 @@ impl Mux { terminal_notifications, notification_ledger, } = restore_public_projections(&state, registry.public_projections()?)?; + let agent_roster = restore_agent_roster(®istry)?; + // The roster is the durable source for adapter identity. Keep the + // legacy record cache in sync for compatibility with direct hook + // projection paths and older callers that inspect it during startup. + let mut agent_records = agent_records; + for (terminal_id, entry) in &agent_roster.roster.entries { + let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) else { continue }; + let record = TerminalAgentRecord { + state: entry.agent_state(), + source: entry.agent_source(), + session: entry.session.clone(), + agent: entry.agent.clone(), + updated_at_ms: entry.updated_at_ms, + }; + agent_records.insert(terminal_id, record); + } let journal_producers = registry.journal_producer_manifests()?; let session_public_id = registry.session_id().clone(); let journal_kernel = crate::journal_kernel::JournalKernel::new( @@ -2470,6 +2535,7 @@ impl Mux { default_colors: Mutex::new(default_colors), durable_terminal_defaults: AtomicBool::new(has_terminal_defaults), sidebar_plugin: Mutex::new(SidebarPluginRuntime::default()), + agent_roster: Mutex::new(agent_roster), agent_records: Mutex::new(agent_records), agent_hook_fences: Mutex::new(agent_hook_fences), placement_notifications: Mutex::new(HashMap::new()), @@ -2587,6 +2653,14 @@ impl Mux { for (producer_id, origin, key, sequence, ingress) in pending { match self.apply_agent_hook_record(&ingress, sequence) { Ok(()) => { + self.fold_agent_roster( + &ingress, + &crate::JournalAppendCommit { + sequence, + event_id: String::new(), + replayed: true, + }, + ); if self .workspace_registry .lock() @@ -5426,7 +5500,10 @@ impl Mux { // process can crash after the durable journal commit and before the // in-memory/resource projection update. The sequence guard makes this // a no-op for already-applied events while allowing restart repair. + let lifecycle_event = agent_state_for_hook_kind(&ingress.kind).is_some(); + let mut should_fold_roster = !lifecycle_event; if let Err(error) = self.apply_agent_hook_record(ingress, commit.sequence) { + should_fold_roster = false; if agent_hook_terminal_gone(&error) { if let Err(_bookkeeping_error) = self .workspace_registry @@ -5453,17 +5530,156 @@ impl Mux { ); } } - } else if let Err(_bookkeeping_error) = self + } else { + if lifecycle_event { + // A stale or fenced hook can be accepted as a durable journal + // replay without advancing the projection watermark. Fold it + // only when this commit was the projection that advanced the + // watermark, so the roster cannot resurrect old state. + should_fold_roster = self + .workspace_registry + .lock() + .unwrap() + .agent_hook_apply_cursor() + .ok() + .is_some_and(|cursor| cursor == commit.sequence); + } + if let Err(_bookkeeping_error) = self + .workspace_registry + .lock() + .unwrap() + .clear_agent_hook_pending(&ingress.producer_id, origin, idempotency_key) + { + self.report_internal_diagnostic( + "agent hook projection applied; retry bookkeeping cleanup deferred", + ); + } + } + if should_fold_roster { + self.fold_agent_roster(ingress, &commit); + } + Ok(commit) + } + + /// Fold the exact journal row after it is committed. The journal row is + /// used instead of the ingress value so its committed timestamp is also + /// the timestamp used during restart replay. Projection compatibility is + /// kept in the existing record cache; the roster snapshot remains the + /// durable source for lifecycle state and adapter identity. + fn fold_agent_roster( + &self, + ingress: &crate::JournalIngress, + commit: &crate::JournalAppendCommit, + ) { + use crate::journal_reducers::{ + AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, RosterEvent, + }; + if ingress.producer_id != crate::agent_hooks::AGENT_HOOK_PRODUCER_ID { + return; + } + let record = match self .workspace_registry .lock() .unwrap() - .clear_agent_hook_pending(&ingress.producer_id, origin, idempotency_key) + .session_journal_after(commit.sequence.saturating_sub(1), 1) { - self.report_internal_diagnostic( - "agent hook projection applied; retry bookkeeping cleanup deferred", - ); + Ok(page) => page.records.into_iter().find(|record| record.sequence == commit.sequence), + Err(error) => { + eprintln!("cmux-tui: reading a committed agent event back failed: {error}"); + return; + } + }; + let Some(record) = record else { return }; + let (deltas, cursor, snapshot) = { + let mut host = self.agent_roster.lock().unwrap(); + if commit.sequence <= host.cursor { + return; + } + let deltas = host.roster.apply(&RosterEvent::from_record(&record)); + host.cursor = host.cursor.max(commit.sequence); + (deltas, host.cursor, host.roster.snapshot().to_string()) + }; + if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( + AGENT_ROSTER_REDUCER_ID, + AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ) { + eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); + } + for delta in deltas { + self.apply_roster_delta_to_record_cache(delta); + } + } + + fn apply_roster_delta_to_record_cache(&self, delta: crate::journal_reducers::RosterDelta) { + use crate::journal_reducers::RosterDelta; + let mut records = self.agent_records.lock().unwrap(); + match delta { + RosterDelta::Upsert { terminal_id, entry } => { + let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) else { return }; + records.insert( + terminal_id, + TerminalAgentRecord { + state: entry.agent_state(), + source: entry.agent_source(), + session: entry.session, + agent: entry.agent, + updated_at_ms: entry.updated_at_ms, + }, + ); + } + RosterDelta::Remove { terminal_id } => { + if let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) { + records.remove(&terminal_id); + } + } + } + } + + /// Echo direct reports into the journal so a restart can rebuild the + /// socket-owned roster entry from the same durable event stream. + fn append_agent_report_echo( + &self, + terminal_id: &TerminalPublicId, + state: AgentState, + source: AgentSource, + session: Option<&str>, + updated_at_ms: u64, + ) { + use crate::journal_reducers::{SOCKET_REPORT_ADAPTER, SOCKET_REPORT_NATIVE_EVENT}; + let ingress = crate::JournalIngress { + producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), + manifest_version: crate::agent_hooks::AGENT_HOOK_MANIFEST_VERSION, + kind: "agent.state.changed".into(), + schema_version: 1, + occurred_at_ms: None, + subjects: vec![crate::JournalSubject { + kind: "terminal".into(), + id: terminal_id.to_string(), + }], + sensitivity: Some(crate::JournalSensitivity::Sensitive), + payload: serde_json::json!({ + "format": crate::agent_hooks::AGENT_HOOK_FORMAT, + "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, + "native_event": SOCKET_REPORT_NATIVE_EVENT, + "normalized": { + "state": state.as_str(), + "source": source.as_str(), + "source_session": session, + "updated_at_ms": updated_at_ms.to_string(), + }, + "native": {}, + }), + causation_id: None, + correlation_id: None, + }; + let idempotency_key = + format!("agent-report-echo-{}", crate::workspace_registry::new_uuid_v4()); + if let Err(error) = self.append_journal_ingress(&ingress, "agent-report", &idempotency_key) + { + eprintln!("cmux-tui: journaling an agent report for {terminal_id} failed: {error}"); } - Ok(commit) } /// Committed agent hook events double as the live agent-status feed: @@ -5570,6 +5786,12 @@ impl Mux { applied_sequence: sequence, ended: state == AgentState::Done, }; + let agent_adapter = ingress + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + .map(str::to_owned); self.report_agent_with_sequence_lock( surface, state, @@ -5578,6 +5800,8 @@ impl Mux { true, Some(hook_state), Some(sequence), + AgentReportOrigin::RosterFold, + agent_adapter, )?; fences.insert( terminal_id.clone(), @@ -8949,7 +9173,17 @@ impl Mux { source: AgentSource, session: Option, ) -> anyhow::Result { - self.report_agent_with_sequence_lock(surface, state, source, session, false, None, None) + self.report_agent_with_sequence_lock( + surface, + state, + source, + session, + false, + None, + None, + AgentReportOrigin::Direct, + None, + ) } fn report_agent_with_sequence_lock( @@ -8961,6 +9195,8 @@ impl Mux { sequence_lock_held: bool, hook_state: Option, journal_sequence: Option, + origin: AgentReportOrigin, + agent_adapter: Option, ) -> anyhow::Result { let mutation = WorkspaceMutation::new( format!("raw-agent-{}", crate::workspace_registry::new_uuid_v4()), @@ -8984,6 +9220,8 @@ impl Mux { sequence_lock_held, hook_state.as_ref(), journal_sequence, + origin, + agent_adapter, )?; let record = record.context("fresh raw agent report unexpectedly replayed")?; if source != AgentSource::Hook { @@ -9024,6 +9262,8 @@ impl Mux { false, None, None, + AgentReportOrigin::Direct, + None, ); if result.is_ok() && source != AgentSource::Hook { let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); @@ -9044,6 +9284,8 @@ impl Mux { sequence_lock_held: bool, hook_state: Option<&crate::workspace_registry::AgentHookProjectionState>, journal_sequence: Option, + origin: AgentReportOrigin, + agent_adapter: Option, ) -> anyhow::Result<(ResourcePatchCommit, Option)> { // Hook replay already owns this guard to serialize sequence checks and // projection commits. Other report sources acquire it before the @@ -9179,7 +9421,7 @@ impl Mux { state: agent_state, source, session: source_session, - agent: None, + agent: agent_adapter, updated_at_ms: now, }, }; @@ -9281,6 +9523,15 @@ impl Mux { agent: agent.agent.as_deref().map(Arc::from), updated_at_ms: agent.updated_at_ms, }); + if origin == AgentReportOrigin::Direct && agent.source == AgentSource::Socket { + self.append_agent_report_echo( + &agent.terminal_id, + agent.state, + agent.source, + agent.session.as_deref(), + agent.updated_at_ms, + ); + } } Ok((commit, Some(agent))) } @@ -9312,6 +9563,22 @@ impl Mux { // The registry guard is dropped before acquiring the fence guard. self.agent_hook_fences.lock().unwrap().remove(terminal_id); self.agent_records.lock().unwrap().remove(terminal_id); + let retired = { + let mut host = self.agent_roster.lock().unwrap(); + host.roster + .retire_terminal(terminal_id.as_str()) + .then(|| (host.cursor, host.roster.snapshot().to_string())) + }; + if let Some((cursor, snapshot)) = retired + && let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ) + { + eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); + } self.terminal_notifications.lock().unwrap().remove(terminal_id); } From 1823ffa19fac8f41a015086fe669b111f056e767 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:48:11 -0700 Subject: [PATCH 013/133] test(tui): fix roster restore surface selector --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b8f6b76035bf..ff0893e451c8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22502,8 +22502,8 @@ mod tests { true, ) .unwrap(); - let surface = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); - let records = reopened.list_agents(Some(surface.id), None); + let surface_id = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); + let records = reopened.list_agents(Some(surface_id), None); assert_eq!(records.len(), 1); assert_eq!(records[0].agent.as_deref(), Some("claude")); reopened.shutdown(); From a1159016abb4a49021715c2eaf228a9e688c5bd8 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:04:10 -0700 Subject: [PATCH 014/133] test(tui): cover ordered roster replay and snapshot recovery --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 112 +++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ff0893e451c8..4dd0fef5d9f4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22511,6 +22511,118 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn roster_fold_replays_the_committed_prefix_before_advancing_cursor() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-prefix-{}", crate::workspace_registry::new_uuid_v4())); + let mux = Mux::open_persistent("roster-prefix", SurfaceOptions::default(), &root).unwrap(); + let first = mux.new_workspace(None, None).unwrap(); + let second = mux.new_workspace(None, None).unwrap(); + let first_terminal = first.terminal_public_id().cloned().expect("first terminal"); + let second_terminal = second.terminal_public_id().cloned().expect("second terminal"); + let socket_event = + |terminal_id: &TerminalPublicId, state: AgentState| crate::JournalIngress { + producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), + manifest_version: crate::agent_hooks::AGENT_HOOK_MANIFEST_VERSION, + kind: "agent.state.changed".into(), + schema_version: 1, + occurred_at_ms: None, + subjects: vec![crate::JournalSubject { + kind: "terminal".into(), + id: terminal_id.to_string(), + }], + sensitivity: Some(crate::JournalSensitivity::Sensitive), + payload: serde_json::json!({ + "format": crate::agent_hooks::AGENT_HOOK_FORMAT, + "adapter": {"id": crate::journal_reducers::SOCKET_REPORT_ADAPTER, "version": 1}, + "native_event": crate::journal_reducers::SOCKET_REPORT_NATIVE_EVENT, + "normalized": { + "state": state.as_str(), + "source": "socket", + "source_session": "prefix-test", + "updated_at_ms": "42", + }, + "native": {}, + }), + causation_id: None, + correlation_id: None, + }; + let first_event = socket_event(&first_terminal, AgentState::Working); + let second_event = socket_event(&second_terminal, AgentState::Blocked); + let first_commit = { + let validated = mux.journal_kernel.validate_ingress(&first_event).unwrap(); + mux.workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&first_event, &validated, "test", "roster-prefix-1") + .unwrap() + }; + let second_commit = { + let validated = mux.journal_kernel.validate_ingress(&second_event).unwrap(); + mux.workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&second_event, &validated, "test", "roster-prefix-2") + .unwrap() + }; + + mux.fold_agent_roster(&second_event, &second_commit); + let host = mux.agent_roster.lock().unwrap(); + assert_eq!(host.cursor, second_commit.sequence); + assert!(host.roster.entries.contains_key(first_terminal.as_str())); + assert!(host.roster.entries.contains_key(second_terminal.as_str())); + assert!(second_commit.sequence > first_commit.sequence); + drop(host); + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn malformed_roster_snapshot_replays_from_the_journal_head() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-corrupt-{}", crate::workspace_registry::new_uuid_v4())); + let session = "roster-corrupt"; + let (terminal_id, sequence) = { + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + let mux = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"corrupt-snapshot"}), + ) + .unwrap(); + let commit = mux.append_journal_ingress(&ingress, "test", "corrupt-snapshot").unwrap(); + mux.shutdown(); + drop(mux); + (terminal_id, commit.sequence) + }; + + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + registry + .put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + sequence, + "{malformed", + ) + .unwrap(); + let restored = restore_agent_roster(®istry).unwrap(); + assert_eq!(restored.cursor, sequence); + assert_eq!(restored.roster.entries[terminal_id.as_str()].agent.as_deref(), Some("claude")); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn stale_same_terminal_hook_sequence_cannot_overwrite_newer_state() { let mux = test_mux(); From 9d050addf297eace95b955acf5e509f43bbab04e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:09:09 -0700 Subject: [PATCH 015/133] fix(tui): fold roster journal prefix safely --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 120 +++++++++++++++++------ 1 file changed, 90 insertions(+), 30 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4dd0fef5d9f4..a7ad44e9dc22 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1101,9 +1101,22 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { - AgentRosterHost { roster: AgentRoster::restore(&snapshot).unwrap_or_default(), cursor } + match AgentRoster::restore(&snapshot) { + Some(roster) => { + snapshot_valid = true; + AgentRosterHost { roster, cursor } + } + None => { + eprintln!("cmux-tui: invalid agent roster snapshot; replaying journal"); + AgentRosterHost::default() + } + } } _ => AgentRosterHost::default(), }; @@ -1118,7 +1131,7 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result, agent_roster: Mutex, + /// Serialize journal folds so a later commit cannot advance the reducer + /// cursor past an earlier committed record. + agent_roster_fold: Mutex<()>, agent_records: Mutex>, agent_hook_fences: Mutex>, /// Nonterminal notifications remain placement-local. Terminal unread @@ -2536,6 +2552,7 @@ impl Mux { durable_terminal_defaults: AtomicBool::new(has_terminal_defaults), sidebar_plugin: Mutex::new(SidebarPluginRuntime::default()), agent_roster: Mutex::new(agent_roster), + agent_roster_fold: Mutex::new(()), agent_records: Mutex::new(agent_records), agent_hook_fences: Mutex::new(agent_hook_fences), placement_notifications: Mutex::new(HashMap::new()), @@ -5577,38 +5594,77 @@ impl Mux { if ingress.producer_id != crate::agent_hooks::AGENT_HOOK_PRODUCER_ID { return; } - let record = match self - .workspace_registry - .lock() - .unwrap() - .session_journal_after(commit.sequence.saturating_sub(1), 1) - { - Ok(page) => page.records.into_iter().find(|record| record.sequence == commit.sequence), - Err(error) => { - eprintln!("cmux-tui: reading a committed agent event back failed: {error}"); + // Journal callbacks can complete out of order. Serialize folds and + // replay the contiguous committed prefix, so a later sequence never + // advances the cursor past an earlier event that has not been folded. + let _fold = self.agent_roster_fold.lock().unwrap(); + loop { + let (cursor, mut candidate) = { + let host = self.agent_roster.lock().unwrap(); + if commit.sequence <= host.cursor { + return; + } + (host.cursor, host.roster.clone()) + }; + let page = + match self.workspace_registry.lock().unwrap().session_journal_after(cursor, 512) { + Ok(page) => page, + Err(error) => { + // The durable event remains available for a later fold or + // restart. Leave the cursor unchanged when the read fails. + eprintln!("cmux-tui: reading committed agent events back failed: {error}"); + return; + } + }; + if page.records.is_empty() { return; } - }; - let Some(record) = record else { return }; - let (deltas, cursor, snapshot) = { - let mut host = self.agent_roster.lock().unwrap(); - if commit.sequence <= host.cursor { + + let mut next_cursor = cursor; + let mut deltas = Vec::new(); + for record in page.records { + if record.sequence <= next_cursor { + continue; + } + // Do not consume a record committed after the callback's + // target. A later callback will fold that suffix. + if record.sequence > commit.sequence { + break; + } + deltas.extend(candidate.apply(&RosterEvent::from_record(&record))); + next_cursor = record.sequence; + } + if next_cursor == cursor { + // The writer acknowledged a commit that the reader cannot see + // yet. Keep the cursor unchanged so another callback retries. + return; + } + let snapshot = candidate.snapshot().to_string(); + if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( + AGENT_ROSTER_REDUCER_ID, + AGENT_ROSTER_REDUCER_VERSION, + next_cursor, + &snapshot, + ) { + eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); + return; + } + { + let mut host = self.agent_roster.lock().unwrap(); + if host.cursor != cursor { + // The fold gate should make this impossible. Preserve a + // newer host state if a future writer violates that rule. + return; + } + host.roster = candidate; + host.cursor = next_cursor; + } + for delta in deltas { + self.apply_roster_delta_to_record_cache(delta); + } + if commit.sequence <= next_cursor { return; } - let deltas = host.roster.apply(&RosterEvent::from_record(&record)); - host.cursor = host.cursor.max(commit.sequence); - (deltas, host.cursor, host.roster.snapshot().to_string()) - }; - if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( - AGENT_ROSTER_REDUCER_ID, - AGENT_ROSTER_REDUCER_VERSION, - cursor, - &snapshot, - ) { - eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); - } - for delta in deltas { - self.apply_roster_delta_to_record_cache(delta); } } @@ -9563,6 +9619,10 @@ impl Mux { // The registry guard is dropped before acquiring the fence guard. self.agent_hook_fences.lock().unwrap().remove(terminal_id); self.agent_records.lock().unwrap().remove(terminal_id); + // Retirement and journal folding share one lifecycle gate. This keeps + // a concurrent fold from installing a candidate snapshot after the + // terminal has been removed. + let _fold = self.agent_roster_fold.lock().unwrap(); let retired = { let mut host = self.agent_roster.lock().unwrap(); host.roster From 147a70a5c5b8d14ebb9cf5b05dd96534026050bc Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:10:53 -0700 Subject: [PATCH 016/133] fix(tui): require a visible roster fold target --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a7ad44e9dc22..9cedbb5d24b6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5622,6 +5622,7 @@ impl Mux { let mut next_cursor = cursor; let mut deltas = Vec::new(); + let mut reached_target = false; for record in page.records { if record.sequence <= next_cursor { continue; @@ -5633,8 +5634,12 @@ impl Mux { } deltas.extend(candidate.apply(&RosterEvent::from_record(&record))); next_cursor = record.sequence; + if next_cursor == commit.sequence { + reached_target = true; + break; + } } - if next_cursor == cursor { + if !reached_target { // The writer acknowledged a commit that the reader cannot see // yet. Keep the cursor unchanged so another callback retries. return; From 58e3ffd019975255afd394ff4c2bbae46739456a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:51:08 -0700 Subject: [PATCH 017/133] fix(tui): fence retired roster terminals and coalesce echoes --- .../cmux-tui-core/src/journal_reducers.rs | 86 ++++++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 214 ++++++++++++++++-- 2 files changed, 275 insertions(+), 25 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 8ca3a6c3f2c5..dd677aece4d8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -16,7 +16,7 @@ //! echo event after its direct projection commit), so the roster never has //! a second writer to diverge from. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -78,6 +78,9 @@ fn state_for_hook_kind(kind: &str) -> Option { /// `SessionJournalRecord`s during tail replay, with identical semantics so /// both paths fold to the same state. pub(crate) struct RosterEvent<'a> { + /// Monotonic journal sequence. Synthetic test events may use zero; + /// persisted records always carry their committed sequence. + pub(crate) sequence: u64, pub(crate) producer_id: &'a str, pub(crate) kind: &'a str, pub(crate) subjects: &'a [JournalSubject], @@ -88,6 +91,7 @@ pub(crate) struct RosterEvent<'a> { impl<'a> RosterEvent<'a> { pub(crate) fn from_record(record: &'a SessionJournalRecord) -> Self { Self { + sequence: record.sequence, producer_id: &record.producer.id, kind: &record.kind, subjects: &record.subjects, @@ -152,6 +156,11 @@ pub(crate) enum RosterDelta { #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub(crate) struct AgentRoster { pub(crate) entries: HashMap, + /// Terminals removed from the resource tree remain fenced in the durable + /// reducer. The value is the highest committed journal cursor observed at + /// retirement, so delayed records from that terminal cannot recreate it. + #[serde(default)] + retired_terminals: BTreeMap, } impl AgentRoster { @@ -163,6 +172,14 @@ impl AgentRoster { return Vec::new(); } let Some(terminal_id) = event.terminal_id() else { return Vec::new() }; + if let Some(retired_at) = self.retired_terminals.get(terminal_id).copied() { + // A terminal can only become live again through a newer explicit + // session start. All other delayed records remain fenced forever. + if event.kind != "agent.session.started" || event.sequence <= retired_at { + return Vec::new(); + } + self.retired_terminals.remove(terminal_id); + } let (state, source, session, agent, updated_at_ms) = if event.adapter_id() == Some(SOCKET_REPORT_ADAPTER) { // Socket echo: explicit state and timestamp carried in the @@ -200,6 +217,10 @@ impl AgentRoster { // An ended agent leaves the roster entirely; the done state is // still committed to the durable projection by the host so // history and remote caches converge. + self.retired_terminals + .entry(terminal_id.to_string()) + .and_modify(|retired_at| *retired_at = (*retired_at).max(event.sequence)) + .or_insert(event.sequence); return if self.entries.remove(terminal_id).is_some() { vec![RosterDelta::Remove { terminal_id: terminal_id.to_string() }] } else { @@ -236,8 +257,24 @@ impl AgentRoster { /// Drop a terminal that left the session (tab closed, terminal /// tombstoned). Terminal lifecycle does not flow through `agent.*` /// events yet, so the host retires entries explicitly. - pub(crate) fn retire_terminal(&mut self, terminal_id: &str) -> bool { - self.entries.remove(terminal_id).is_some() + pub(crate) fn retire_terminal(&mut self, terminal_id: &str, retired_at: u64) -> bool { + let removed_entry = self.entries.remove(terminal_id).is_some(); + match self.retired_terminals.entry(terminal_id.to_string()) { + std::collections::btree_map::Entry::Occupied(mut entry) => { + let previous = *entry.get(); + let effective = previous.max(retired_at); + entry.insert(effective); + removed_entry || effective != previous + } + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(retired_at); + true + } + } + } + + pub(crate) fn is_retired(&self, terminal_id: &str) -> bool { + self.retired_terminals.contains_key(terminal_id) } pub(crate) fn snapshot(&self) -> Value { @@ -261,6 +298,7 @@ mod tests { payload: &'a Value, ) -> RosterEvent<'a> { RosterEvent { + sequence, producer_id: AGENT_HOOK_PRODUCER_ID, kind, subjects, @@ -406,4 +444,46 @@ mod tests { assert!(roster.apply(&foreign).is_empty()); assert!(!roster.entries.is_empty()); } + + #[test] + fn retired_terminal_rejects_late_events_until_a_new_session_starts() { + let subjects = terminal_subject("term_a"); + let payload = json!({}); + let mut roster = AgentRoster::default(); + + roster.apply(&hook_event(1, "agent.session.started", &subjects, &payload)); + assert!(roster.retire_terminal("term_a", 1)); + + // Events from the retired terminal cannot recreate a live entry, + // including events committed after the retirement cursor. + assert!(roster.apply(&hook_event(2, "agent.turn.started", &subjects, &payload)).is_empty()); + assert!(!roster.entries.contains_key("term_a")); + + // Only a newer explicit session start clears the tombstone. + assert!( + roster.apply(&hook_event(1, "agent.session.started", &subjects, &payload)).is_empty() + ); + let deltas = roster.apply(&hook_event(2, "agent.session.started", &subjects, &payload)); + assert_eq!(deltas.len(), 1); + assert_eq!(roster.entries["term_a"].state, "idle"); + } + + #[test] + fn retirement_cursor_only_moves_forward() { + let subjects = terminal_subject("term_a"); + let payload = json!({}); + let mut roster = AgentRoster::default(); + + roster.apply(&hook_event(1, "agent.session.started", &subjects, &payload)); + assert!(roster.retire_terminal("term_a", 9)); + assert!(!roster.retire_terminal("term_a", 4)); + + assert!(roster.apply(&hook_event(8, "agent.turn.started", &subjects, &payload)).is_empty()); + assert!( + roster.apply(&hook_event(10, "agent.turn.started", &subjects, &payload)).is_empty() + ); + let deltas = roster.apply(&hook_event(10, "agent.session.started", &subjects, &payload)); + assert_eq!(deltas.len(), 1); + assert!(!roster.is_retired("term_a")); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 9cedbb5d24b6..062b3a032dc8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5698,6 +5698,61 @@ impl Mux { } } + /// Return a stable echo receipt for a semantic socket-state transition. + /// Repeated polls of the same state do not append another journal row; + /// the prior reducer state and cursor distinguish a later transition back + /// to that state from a duplicate poll. + fn agent_report_echo_key( + &self, + terminal_id: &TerminalPublicId, + state: AgentState, + source: AgentSource, + session: Option<&str>, + ) -> Option { + let (cursor, previous) = { + let host = self.agent_roster.lock().unwrap(); + let previous = host.roster.entries.get(terminal_id.as_str()).map(|entry| { + ( + entry.state.clone(), + entry.source.clone(), + entry.session.clone(), + entry.agent.clone(), + entry.updated_at_ms, + ) + }); + (host.cursor, previous) + }; + let unchanged = previous.as_ref().is_some_and( + |(previous_state, previous_source, previous_session, _, _)| { + previous_state == state.as_str() + && previous_source == source.as_str() + && previous_session.as_deref() == session + }, + ); + if unchanged { + return None; + } + let previous = previous + .map(|(state, source, session, agent, updated_at_ms)| { + format!("{state}|{source}|{:?}|{:?}|{updated_at_ms}", session, agent) + }) + .unwrap_or_default(); + let material = format!( + "agent-report-echo-v1|{terminal_id}|{}|{}|{:?}|{cursor}|{previous}", + state.as_str(), + source.as_str(), + session, + ); + let digest = Sha256::digest(material.as_bytes()); + let mut key = String::with_capacity("agent-report-echo-".len() + digest.len() * 2); + key.push_str("agent-report-echo-"); + for byte in digest { + use std::fmt::Write as _; + write!(key, "{byte:02x}").expect("writing to a String cannot fail"); + } + Some(key) + } + /// Echo direct reports into the journal so a restart can rebuild the /// socket-owned roster entry from the same durable event stream. fn append_agent_report_echo( @@ -5707,6 +5762,7 @@ impl Mux { source: AgentSource, session: Option<&str>, updated_at_ms: u64, + idempotency_key: &str, ) { use crate::journal_reducers::{SOCKET_REPORT_ADAPTER, SOCKET_REPORT_NATIVE_EVENT}; let ingress = crate::JournalIngress { @@ -5735,10 +5791,7 @@ impl Mux { causation_id: None, correlation_id: None, }; - let idempotency_key = - format!("agent-report-echo-{}", crate::workspace_registry::new_uuid_v4()); - if let Err(error) = self.append_journal_ingress(&ingress, "agent-report", &idempotency_key) - { + if let Err(error) = self.append_journal_ingress(&ingress, "agent-report", idempotency_key) { eprintln!("cmux-tui: journaling an agent report for {terminal_id} failed: {error}"); } } @@ -9397,6 +9450,9 @@ impl Mux { (surface, (*terminal_id).clone()) } }; + if self.agent_roster.lock().unwrap().roster.is_retired(terminal_id.as_str()) { + anyhow::bail!("terminal {terminal_id} has been retired"); + } let mut direct_hook_state = None; if source != AgentSource::Hook { let ended_fence = sequence_guard @@ -9584,13 +9640,22 @@ impl Mux { agent: agent.agent.as_deref().map(Arc::from), updated_at_ms: agent.updated_at_ms, }); - if origin == AgentReportOrigin::Direct && agent.source == AgentSource::Socket { + if origin == AgentReportOrigin::Direct + && agent.source == AgentSource::Socket + && let Some(idempotency_key) = self.agent_report_echo_key( + &agent.terminal_id, + agent.state, + agent.source, + agent.session.as_deref(), + ) + { self.append_agent_report_echo( &agent.terminal_id, agent.state, agent.source, agent.session.as_deref(), agent.updated_at_ms, + &idempotency_key, ); } } @@ -9614,36 +9679,51 @@ impl Mux { } fn purge_terminal_side_tables(&self, terminal_id: &TerminalPublicId) { - let pending_cleanup = { + // Retirement and journal folding share one lifecycle gate. This keeps + // a concurrent fold from installing a candidate snapshot after the + // terminal has been removed. The tombstone is persisted before the + // compatibility cache is cleared. + let _fold = self.agent_roster_fold.lock().unwrap(); + let (journal_cursor, pending_cleanup) = { let mut registry = self.workspace_registry.lock().unwrap(); - registry.purge_agent_hook_pending_for_terminal(terminal_id) + let pending_cleanup = registry.purge_agent_hook_pending_for_terminal(terminal_id); + let journal_cursor = match registry.session_journal_after(0, 1) { + Ok(page) => page.head_sequence, + Err(error) => { + eprintln!( + "cmux-tui: reading the journal head before terminal retirement failed: {error}" + ); + return; + } + }; + (journal_cursor, pending_cleanup) }; if pending_cleanup.is_err() { self.report_internal_diagnostic("terminal agent hook cleanup deferred"); } - // The registry guard is dropped before acquiring the fence guard. - self.agent_hook_fences.lock().unwrap().remove(terminal_id); - self.agent_records.lock().unwrap().remove(terminal_id); - // Retirement and journal folding share one lifecycle gate. This keeps - // a concurrent fold from installing a candidate snapshot after the - // terminal has been removed. - let _fold = self.agent_roster_fold.lock().unwrap(); let retired = { let mut host = self.agent_roster.lock().unwrap(); - host.roster - .retire_terminal(terminal_id.as_str()) - .then(|| (host.cursor, host.roster.snapshot().to_string())) + let previous = host.roster.clone(); + let retired = host.roster.retire_terminal(terminal_id.as_str(), journal_cursor); + retired.then(|| (previous, host.cursor, host.roster.snapshot().to_string())) }; - if let Some((cursor, snapshot)) = retired - && let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( + if let Some((previous, cursor, snapshot)) = retired { + let persisted = self.workspace_registry.lock().unwrap().put_journal_reducer_state( crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, cursor, &snapshot, - ) - { - eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); + ); + if let Err(error) = persisted { + // Keep the live/cache state intact until the tombstone is + // durable. A later close/retry can then persist it safely. + self.agent_roster.lock().unwrap().roster = previous; + eprintln!("cmux-tui: persisting the agent roster tombstone failed: {error}"); + return; + } } + self.agent_hook_fences.lock().unwrap().remove(terminal_id); + self.agent_records.lock().unwrap().remove(terminal_id); self.terminal_notifications.lock().unwrap().remove(terminal_id); } @@ -22688,6 +22768,96 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn retired_terminal_cannot_be_resurrected_by_a_delayed_journal_event() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-retired-{}", crate::workspace_registry::new_uuid_v4())); + let session = "roster-retired"; + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let event = crate::JournalIngress { + producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), + manifest_version: crate::agent_hooks::AGENT_HOOK_MANIFEST_VERSION, + kind: "agent.turn.started".into(), + schema_version: 1, + occurred_at_ms: None, + subjects: vec![crate::JournalSubject { + kind: "terminal".into(), + id: terminal_id.to_string(), + }], + sensitivity: Some(crate::JournalSensitivity::Sensitive), + payload: serde_json::json!({ + "format": crate::agent_hooks::AGENT_HOOK_FORMAT, + "adapter": {"id": "claude", "version": 1}, + "native_event": "UserPromptSubmit", + "normalized": {}, + "native": {}, + }), + causation_id: None, + correlation_id: None, + }; + let validated = mux.journal_kernel.validate_ingress(&event).unwrap(); + let commit = mux + .workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&event, &validated, "retired-test", "retired-event") + .unwrap(); + + // Simulate terminal teardown before the delayed reducer callback. + mux.purge_terminal_side_tables(&terminal_id); + mux.fold_agent_roster(&event, &commit); + assert!(mux.list_agents(None, None).is_empty()); + assert!(mux.agent_roster.lock().unwrap().roster.entries.is_empty()); + + mux.shutdown(); + drop(mux); + let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + assert!(reopened.list_agents(None, None).is_empty()); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn repeated_socket_reports_coalesce_identical_journal_echoes() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-echo-{}", crate::workspace_registry::new_uuid_v4())); + let mux = Mux::open_persistent("roster-echo", SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let socket_echoes = || { + mux.session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count() + }; + + assert_eq!(socket_echoes(), 0); + mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) + .unwrap(); + assert_eq!(socket_echoes(), 1); + mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) + .unwrap(); + assert_eq!(socket_echoes(), 1); + mux.report_agent(surface.id, AgentState::Blocked, AgentSource::Socket, Some("poll".into())) + .unwrap(); + assert_eq!(socket_echoes(), 2); + + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn stale_same_terminal_hook_sequence_cannot_overwrite_newer_state() { let mux = test_mux(); From 8171dbdfb10eb1cb82fca5c348c5028fcbe4cf93 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:58:34 -0700 Subject: [PATCH 018/133] fix(tui): keep roster tombstones at terminal retirement --- cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index dd677aece4d8..088840718df5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -217,10 +217,6 @@ impl AgentRoster { // An ended agent leaves the roster entirely; the done state is // still committed to the durable projection by the host so // history and remote caches converge. - self.retired_terminals - .entry(terminal_id.to_string()) - .and_modify(|retired_at| *retired_at = (*retired_at).max(event.sequence)) - .or_insert(event.sequence); return if self.entries.remove(terminal_id).is_some() { vec![RosterDelta::Remove { terminal_id: terminal_id.to_string() }] } else { From b0356d4b9becc30c20510038f881fcc9bfb8104f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 03:53:35 -0700 Subject: [PATCH 019/133] fix(tui): preserve agent retirement cleanup errors --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 53 +++++++++++++------ .../src/mux/resource_topology.rs | 13 ++++- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 062b3a032dc8..c79e513a4c78 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -8930,8 +8930,11 @@ impl Mux { } let had_runtime = runtime.is_some(); if let Some(runtime) = runtime { - self.purge_terminal_runtime_side_tables(&runtime); + let purge_result = self.purge_terminal_runtime_side_tables(&runtime); self.terminate_terminal_runtime(&runtime); + if purge_result.is_err() { + self.report_internal_diagnostic("terminal agent retirement deferred after close"); + } } if target.is_some() { self.emit(MuxEvent::TreeChanged); @@ -8969,7 +8972,7 @@ impl Mux { for placement in removed { self.purge_surface_side_tables(placement.id); } - self.purge_terminal_runtime_side_tables(surface); + self.purge_terminal_runtime_side_tables(surface)?; if !surface.is_dead() { surface.kill(); } @@ -8987,7 +8990,7 @@ impl Mux { for placement in removed { self.purge_surface_side_tables(placement.id); } - self.purge_terminal_runtime_side_tables(surface); + self.purge_terminal_runtime_side_tables(surface)?; if !surface.is_dead() { surface.kill(); } @@ -9678,7 +9681,7 @@ impl Mux { self.placement_notifications.lock().unwrap().remove(&surface); } - fn purge_terminal_side_tables(&self, terminal_id: &TerminalPublicId) { + fn purge_terminal_side_tables(&self, terminal_id: &TerminalPublicId) -> anyhow::Result<()> { // Retirement and journal folding share one lifecycle gate. This keeps // a concurrent fold from installing a candidate snapshot after the // terminal has been removed. The tombstone is persisted before the @@ -9690,10 +9693,10 @@ impl Mux { let journal_cursor = match registry.session_journal_after(0, 1) { Ok(page) => page.head_sequence, Err(error) => { - eprintln!( - "cmux-tui: reading the journal head before terminal retirement failed: {error}" + self.report_internal_diagnostic( + "terminal agent retirement deferred before journal fence", ); - return; + return Err(error).context("read journal head before terminal retirement"); } }; (journal_cursor, pending_cleanup) @@ -9718,8 +9721,10 @@ impl Mux { // Keep the live/cache state intact until the tombstone is // durable. A later close/retry can then persist it safely. self.agent_roster.lock().unwrap().roster = previous; - eprintln!("cmux-tui: persisting the agent roster tombstone failed: {error}"); - return; + self.report_internal_diagnostic( + "terminal agent retirement deferred before snapshot", + ); + return Err(error).context("persist agent roster tombstone"); } } self.agent_hook_fences.lock().unwrap().remove(terminal_id); @@ -9727,7 +9732,13 @@ impl Mux { self.terminal_notifications.lock().unwrap().remove(terminal_id); } - fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) { + fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) -> anyhow::Result<()> { + if let Some(terminal_id) = runtime.terminal_public_id() { + // Persist the terminal tombstone before clearing any runtime cache. + // A failed journal read or snapshot write must leave all side tables + // available for a later retry. + self.purge_terminal_side_tables(terminal_id)?; + } if let Some(runtime_id) = runtime.terminal_runtime_id() { self.reserved_in_process_terminals.lock().unwrap().remove(&runtime_id); let _cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); @@ -9741,9 +9752,7 @@ impl Mux { } } } - if let Some(terminal_id) = runtime.terminal_public_id() { - self.purge_terminal_side_tables(terminal_id); - } + Ok(()) } pub fn list_agents( @@ -12422,10 +12431,15 @@ impl Mux { for placement in rollback_removed { self.purge_surface_side_tables(placement.id); } - self.purge_terminal_runtime_side_tables(&surface); + let purge_result = self.purge_terminal_runtime_side_tables(&surface); if !surface.is_dead() { surface.kill(); } + if purge_result.is_err() { + self.report_internal_diagnostic( + "terminal agent retirement deferred after attach rollback", + ); + } return Err(error); } }; @@ -15511,7 +15525,11 @@ impl Mux { self.purge_surface_side_tables(placement.id); } for surface in spawned { - self.purge_terminal_runtime_side_tables(&surface); + if let Err(_error) = self.purge_terminal_runtime_side_tables(&surface) { + self.report_internal_diagnostic( + "terminal agent retirement deferred after layout rollback", + ); + } if !surface.is_dead() { surface.kill(); } @@ -22806,7 +22824,7 @@ mod tests { .unwrap(); // Simulate terminal teardown before the delayed reducer callback. - mux.purge_terminal_side_tables(&terminal_id); + mux.purge_terminal_side_tables(&terminal_id).unwrap(); mux.fold_agent_roster(&event, &commit); assert!(mux.list_agents(None, None).is_empty()); assert!(mux.agent_roster.lock().unwrap().roster.entries.is_empty()); @@ -23768,6 +23786,7 @@ mod tests { mux.apply_agent_hook_record(&ingress("SessionStart", Some("ended-session")), 1).unwrap(); mux.apply_agent_hook_record(&ingress("SessionEnd", Some("ended-session")), 2).unwrap(); assert!(mux.list_agents(Some(surface.id), None).is_empty()); + drop(surface); mux.shutdown(); drop(mux); @@ -23854,7 +23873,7 @@ mod tests { crate::workspace_registry::AgentHookRetryClass::Transient, ) .unwrap(); - mux.purge_terminal_side_tables(&terminal_id); + mux.purge_terminal_side_tables(&terminal_id).unwrap(); assert!( mux.workspace_registry .lock() diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index d2e51f7244f2..7793ac83e7ee 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2544,7 +2544,11 @@ impl Mux { self.purge_surface_side_tables(*target); } if let Some(runtime) = effects.runtime.as_ref() { - self.purge_terminal_runtime_side_tables(runtime); + if let Err(_error) = self.purge_terminal_runtime_side_tables(runtime) { + self.report_internal_diagnostic( + "terminal agent retirement deferred for exited terminal", + ); + } runtime.kill(); } drop(effects.removed); @@ -2719,8 +2723,13 @@ impl Mux { } } if let Some(runtime) = effects.terminal_runtime { - self.purge_terminal_runtime_side_tables(&runtime); + let purge_result = self.purge_terminal_runtime_side_tables(&runtime); self.terminate_terminal_runtime(&runtime); + if purge_result.is_err() { + self.report_internal_diagnostic( + "terminal agent retirement deferred after resource close", + ); + } } match effects.tree_publication { ResourceCloseTreePublication::PendingDelta(delta) => { From 0558132c934e98cfbf7038c7d5615958c5452d74 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 04:11:31 -0700 Subject: [PATCH 020/133] fix(tui): return successful retirement cleanup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c79e513a4c78..1eda48ea5b58 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9730,6 +9730,7 @@ impl Mux { self.agent_hook_fences.lock().unwrap().remove(terminal_id); self.agent_records.lock().unwrap().remove(terminal_id); self.terminal_notifications.lock().unwrap().remove(terminal_id); + Ok(()) } fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) -> anyhow::Result<()> { From d9edfb1d697d03f9bee18bc76f49986e2263b6b5 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:31:59 -0700 Subject: [PATCH 021/133] fix(tui): retry terminal agent cleanup on startup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 87 ++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 1eda48ea5b58..881212d9de1d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2617,6 +2617,7 @@ impl Mux { } std::thread::sleep(Duration::from_millis(25)); } + mux.reconcile_retired_terminal_side_tables(); mux.retry_pending_agent_hooks()?; crate::journal_hooks::start(&mux)?; Ok(mux) @@ -2639,6 +2640,49 @@ impl Mux { Ok(()) } + /// A durable terminal tombstone is also the retry receipt for best-effort + /// side-table cleanup. A close reply stays successful after its resource + /// transaction commits; startup retries any stale agent state left by a + /// later cleanup failure. + fn reconcile_retired_terminal_side_tables(&self) { + let mut candidates = HashSet::new(); + candidates.extend(self.agent_records.lock().unwrap().keys().cloned()); + candidates.extend(self.agent_hook_fences.lock().unwrap().keys().cloned()); + candidates.extend(self.terminal_notifications.lock().unwrap().keys().cloned()); + let roster_terminal_ids = + self.agent_roster.lock().unwrap().roster.entries.keys().cloned().collect::>(); + for terminal_id in roster_terminal_ids { + if let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) { + candidates.insert(terminal_id); + } + } + + for terminal_id in candidates { + let retired: anyhow::Result = (|| { + let registry = self.workspace_registry.lock().unwrap(); + let Some(host_id) = registry.terminal_host_id(&terminal_id)? else { + return Ok(true); + }; + Ok(registry + .terminal_record(&host_id)? + .is_none_or(|terminal| terminal.lifecycle == TerminalLifecycle::Tombstoned)) + })(); + match retired { + Ok(true) => { + if self.purge_terminal_side_tables(&terminal_id).is_err() { + self.report_internal_diagnostic( + "retired terminal side-table cleanup remains deferred", + ); + } + } + Ok(false) => {} + Err(_) => self.report_internal_diagnostic( + "retired terminal side-table reconciliation remains deferred", + ), + } + } + } + fn retry_pending_agent_hooks_for_terminal( &self, terminal_id: &TerminalPublicId, @@ -9704,6 +9748,7 @@ impl Mux { if pending_cleanup.is_err() { self.report_internal_diagnostic("terminal agent hook cleanup deferred"); } + pending_cleanup.context("purge pending agent hook receipts")?; let retired = { let mut host = self.agent_roster.lock().unwrap(); let previous = host.roster.clone(); @@ -23885,6 +23930,48 @@ mod tests { ); } + #[test] + fn startup_reconciliation_retries_agent_cleanup_after_durable_close() { + let root = std::env::temp_dir() + .join(format!("cmux-agent-cleanup-retry-{}", crate::workspace_registry::new_uuid_v4())); + let session = "agent-cleanup-retry"; + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"cleanup-retry"}), + ) + .unwrap(); + mux.append_journal_ingress(&ingress, "test", "cleanup-retry-agent").unwrap(); + assert_eq!(mux.list_agents(Some(surface.id), None).len(), 1); + + // Simulate a process loss after the terminal close transaction + // commits but before the normal side-table cleanup runs. + let mutation = WorkspaceMutation::new("cleanup-retry-close", "test").unwrap(); + { + let mut registry = mux.workspace_registry.lock().unwrap(); + let host_id = registry.terminal_host_id(&terminal_id).unwrap().unwrap(); + let incarnation = registry.terminal_record(&host_id).unwrap().unwrap().incarnation; + registry + .close_terminal(&mutation, None, None, &host_id, incarnation.as_deref()) + .unwrap(); + } + assert_eq!(mux.list_agents(Some(surface.id), None).len(), 1); + + mux.shutdown(); + drop(mux); + let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + + assert!(reopened.list_agents(None, None).is_empty()); + assert!(reopened.agent_roster.lock().unwrap().roster.is_retired(terminal_id.as_str())); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn failed_raw_agent_report_rolls_back_projection_memory_revision_and_event() { let mux = test_mux(); From 47974c8d05cf25c1eb194c587df9d7d7ff6aeea7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:32:33 -0700 Subject: [PATCH 022/133] fix(tui): retain ended hook roster fences --- .../cmux-tui-core/src/journal_reducers.rs | 169 +++++++++++++++--- cmux-tui/crates/cmux-tui-core/src/mux.rs | 94 ++++++---- 2 files changed, 200 insertions(+), 63 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 088840718df5..047dbb5d6693 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -27,8 +27,9 @@ use crate::{AgentSource, AgentState, JournalSubject}; pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; /// Bump to discard persisted snapshots and re-fold from the journal head. -/// Version 2 added the agent adapter id to roster entries. -pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 2; +/// Version 2 added the agent adapter id to roster entries. Version 3 retains +/// ended hook fences after their live roster entries are removed. +pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 3; /// The adapter id and native event the socket report path uses for its echo /// journal events. The echo carries the explicit state in `normalized`, so @@ -116,6 +117,13 @@ impl<'a> RosterEvent<'a> { } } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct RosterFence { + session_id: String, + sequence: u64, + ended: bool, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct RosterEntry { pub(crate) state: String, @@ -156,6 +164,11 @@ pub(crate) enum RosterDelta { #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub(crate) struct AgentRoster { pub(crate) entries: HashMap, + /// Hook generation watermarks remain after an ended session leaves the + /// live roster. Socket echoes cannot recreate an ended entry, and delayed + /// events from an old named session cannot cross into a newer lifecycle. + #[serde(default)] + hook_fences: HashMap, /// Terminals removed from the resource tree remain fenced in the durable /// reducer. The value is the highest committed journal cursor observed at /// retirement, so delayed records from that terminal cannot recreate it. @@ -172,37 +185,83 @@ impl AgentRoster { return Vec::new(); } let Some(terminal_id) = event.terminal_id() else { return Vec::new() }; + let socket_echo = event.adapter_id() == Some(SOCKET_REPORT_ADAPTER); if let Some(retired_at) = self.retired_terminals.get(terminal_id).copied() { // A terminal can only become live again through a newer explicit // session start. All other delayed records remain fenced forever. - if event.kind != "agent.session.started" || event.sequence <= retired_at { + if socket_echo || event.kind != "agent.session.started" || event.sequence <= retired_at + { return Vec::new(); } self.retired_terminals.remove(terminal_id); + self.hook_fences.remove(terminal_id); } - let (state, source, session, agent, updated_at_ms) = - if event.adapter_id() == Some(SOCKET_REPORT_ADAPTER) { - // Socket echo: explicit state and timestamp carried in the - // payload, so the roster mirrors the direct projection - // commit exactly. The reporter does not know the agent type. - let Some(state) = event.normalized("state").and_then(agent_state_from_str) else { - return Vec::new(); - }; - // The socket adapter is authoritative about origin. Do not - // trust the user-controlled normalized source field, or it - // could bypass hook-over-socket precedence. - let source = AgentSource::Socket; - let updated_at_ms = event - .normalized("updated_at_ms") - .and_then(|value| value.parse::().ok()) - .unwrap_or(event.committed_at_ms); - let session = event.normalized("source_session").map(str::to_string); - (state, source, session, None, updated_at_ms) - } else { - let Some(state) = state_for_hook_kind(event.kind) else { return Vec::new() }; - let agent = event.adapter_id().map(str::to_string); - (state, AgentSource::Hook, None, agent, event.committed_at_ms) + let (state, source, session, agent, updated_at_ms) = if socket_echo { + // Socket echo: explicit state and timestamp carried in the + // payload, so the roster mirrors the direct projection + // commit exactly. The reporter does not know the agent type. + let Some(state) = event.normalized("state").and_then(agent_state_from_str) else { + return Vec::new(); }; + // The socket adapter is authoritative about origin. Do not + // trust the user-controlled normalized source field, or it + // could bypass hook-over-socket precedence. + let source = AgentSource::Socket; + let updated_at_ms = event + .normalized("updated_at_ms") + .and_then(|value| value.parse::().ok()) + .unwrap_or(event.committed_at_ms); + let session = event.normalized("source_session").map(str::to_string); + if self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) { + return Vec::new(); + } + (state, source, session, None, updated_at_ms) + } else { + let Some(state) = state_for_hook_kind(event.kind) else { return Vec::new() }; + let agent = event.adapter_id().map(str::to_string); + let explicit_session = event + .normalized("agent_session_id") + .filter(|session| !session.is_empty()) + .map(str::to_owned); + let is_session_start = event.kind == "agent.session.started"; + let previous_fence = self.hook_fences.get(terminal_id).cloned(); + // Once an adapter supplies a native identity, a later + // session-less event is ambiguous and cannot attach to that + // generation. + if explicit_session.is_none() + && previous_fence + .as_ref() + .is_some_and(|fence| !fence.session_id.starts_with("legacy:")) + { + return Vec::new(); + } + let session_id = explicit_session + .or_else(|| { + (!is_session_start) + .then(|| previous_fence.as_ref().filter(|fence| !fence.ended)) + .flatten() + .map(|fence| fence.session_id.clone()) + }) + .unwrap_or_else(|| format!("legacy:{terminal_id}:{}", event.sequence)); + if let Some(fence) = previous_fence.as_ref() { + if fence.session_id == session_id { + if fence.ended || event.sequence <= fence.sequence { + return Vec::new(); + } + } else if !is_session_start || !fence.ended || event.sequence <= fence.sequence { + return Vec::new(); + } + } + self.hook_fences.insert( + terminal_id.to_string(), + RosterFence { + session_id, + sequence: event.sequence, + ended: state == AgentState::Done, + }, + ); + (state, AgentSource::Hook, None, agent, event.committed_at_ms) + }; if source == AgentSource::Socket && self .entries @@ -273,6 +332,10 @@ impl AgentRoster { self.retired_terminals.contains_key(terminal_id) } + pub(crate) fn has_ended_hook_fence(&self, terminal_id: &str) -> bool { + self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) + } + pub(crate) fn snapshot(&self) -> Value { serde_json::to_value(self).unwrap_or(Value::Null) } @@ -381,6 +444,62 @@ mod tests { assert_eq!(roster.entries["term_a"].source, "hook"); } + #[test] + fn ended_hook_fence_survives_snapshot_and_rejects_fresh_socket_echoes() { + let subjects = terminal_subject("term_a"); + let old_hook_payload = json!({ + "adapter": {"id": "claude", "version": 1}, + "normalized": {"agent_session_id": "old-hook-session"} + }); + let new_hook_payload = json!({ + "adapter": {"id": "claude", "version": 1}, + "normalized": {"agent_session_id": "new-hook-session"} + }); + let socket_payload = json!({ + "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, + "normalized": { + "state": "working", + "source": "socket", + "source_session": "fresh-socket-session" + } + }); + let mut roster = AgentRoster::default(); + + roster.apply(&hook_event(1, "agent.session.started", &subjects, &old_hook_payload)); + roster.apply(&hook_event(5, "agent.session.ended", &subjects, &old_hook_payload)); + assert!(roster.has_ended_hook_fence("term_a")); + assert!( + roster + .apply(&hook_event(6, "agent.state.changed", &subjects, &socket_payload)) + .is_empty() + ); + + let mut restored = AgentRoster::restore(&roster.snapshot().to_string()).unwrap(); + assert!(restored.has_ended_hook_fence("term_a")); + assert!( + restored + .apply(&hook_event(7, "agent.state.changed", &subjects, &socket_payload)) + .is_empty() + ); + assert!( + restored + .apply(&hook_event(5, "agent.session.started", &subjects, &new_hook_payload)) + .is_empty() + ); + assert!(restored.has_ended_hook_fence("term_a")); + + let deltas = + restored.apply(&hook_event(6, "agent.session.started", &subjects, &new_hook_payload)); + assert_eq!(deltas.len(), 1); + assert!(!restored.has_ended_hook_fence("term_a")); + assert!( + restored + .apply(&hook_event(7, "agent.turn.started", &subjects, &old_hook_payload)) + .is_empty() + ); + assert_eq!(restored.entries["term_a"].state, "idle"); + } + #[test] fn folds_are_idempotent_per_state_and_deterministic_across_replays() { let subjects = terminal_subject("term_a"); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 881212d9de1d..aca08983d7f4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9497,22 +9497,19 @@ impl Mux { (surface, (*terminal_id).clone()) } }; - if self.agent_roster.lock().unwrap().roster.is_retired(terminal_id.as_str()) { + let roster = self.agent_roster.lock().unwrap(); + if roster.roster.is_retired(terminal_id.as_str()) { anyhow::bail!("terminal {terminal_id} has been retired"); } + let durable_hook_end = roster.roster.has_ended_hook_fence(terminal_id.as_str()); + drop(roster); let mut direct_hook_state = None; if source != AgentSource::Hook { let ended_fence = sequence_guard .as_ref() .and_then(|guard| guard.get(&terminal_id)) .filter(|fence| fence.ended); - let fresh_session = source_session.as_deref().filter(|session| { - !session.starts_with("cmux-hook-sequence:") - && !session.starts_with("cmux-hook-ended:") - }); - if ended_fence.is_some_and(|fence| { - fresh_session.is_none_or(|session| session == fence.session_id) - }) { + if durable_hook_end || ended_fence.is_some() { anyhow::bail!("agent_session_ended"); } } else if let Some(fence) = sequence_guard @@ -23468,55 +23465,75 @@ mod tests { } #[test] - fn new_non_hook_session_can_start_after_hook_session_end() { + fn fresh_socket_session_remains_fenced_until_explicit_hook_start() { let root = std::env::temp_dir() .join(format!("cmux-agent-hook-restart-{}", crate::workspace_registry::new_uuid_v4())); let mux = Mux::open_persistent("agent-hook-restart", SurfaceOptions::default(), &root).unwrap(); let surface = mux.new_workspace(None, None).unwrap(); let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); - let hook = |event: &str| { + let hook = |event: &str, session_id: &str| { crate::agent_hooks::agent_hook_journal_ingress( "claude", event, Some(&terminal_id.to_string()), - serde_json::json!({"session_id":"old-hook"}), + serde_json::json!({"session_id":session_id}), ) .unwrap() }; - mux.apply_agent_hook_record(&hook("SessionEnd"), 1).unwrap(); - - mux.report_agent( - surface.id, - AgentState::Working, - AgentSource::Socket, - Some("new-socket-session".into()), - ) - .unwrap(); - let record = &mux.list_agents(Some(surface.id), None)[0]; - assert_eq!(record.source, AgentSource::Socket); - assert_eq!(record.session.as_deref(), Some("new-socket-session")); + mux.append_journal_ingress(&hook("SessionEnd", "old-hook"), "test", "ended-hook").unwrap(); - let public = crate::resource_api::public_session_snapshot(&mux).unwrap(); - assert_eq!(public["agents"][0]["source_session"], "new-socket-session"); - - // A late event from the ended hook session remains fenced. - mux.apply_agent_hook_record(&hook("UserPromptSubmit"), 2).unwrap(); - let record = &mux.list_agents(Some(surface.id), None)[0]; - assert_eq!(record.source, AgentSource::Socket); - assert_eq!(record.session.as_deref(), Some("new-socket-session")); + assert!( + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("new-socket-session".into()), + ) + .is_err() + ); + assert!(mux.list_agents(Some(surface.id), None).is_empty()); mux.shutdown(); drop(mux); let reopened = Mux::open_persistent("agent-hook-restart", SurfaceOptions::default(), &root).unwrap(); - let reopened_record = &reopened.list_agents(Some(surface.id), None)[0]; - assert_eq!(reopened_record.source, AgentSource::Socket); - assert_eq!(reopened_record.session.as_deref(), Some("new-socket-session")); - assert_eq!( - crate::resource_api::public_session_snapshot(&reopened).unwrap()["agents"][0]["source_session"], - "new-socket-session" + let reopened_surface = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); + assert!( + reopened + .report_agent( + reopened_surface, + AgentState::Working, + AgentSource::Socket, + Some("new-socket-session".into()), + ) + .is_err() ); + assert!(reopened.list_agents(Some(reopened_surface), None).is_empty()); + + reopened + .append_journal_ingress(&hook("SessionStart", "new-hook"), "test", "new-hook-start") + .unwrap(); + assert!( + !reopened + .agent_roster + .lock() + .unwrap() + .roster + .has_ended_hook_fence(terminal_id.as_str()) + ); + assert!( + reopened + .report_agent( + reopened_surface, + AgentState::Working, + AgentSource::Socket, + Some("new-socket-session".into()), + ) + .is_ok() + ); + let reopened_record = &reopened.list_agents(Some(reopened_surface), None)[0]; + assert_eq!(reopened_record.source, AgentSource::Hook); reopened.shutdown(); std::fs::remove_dir_all(root).unwrap(); } @@ -23535,7 +23552,8 @@ mod tests { ) .unwrap() }; - mux.apply_agent_hook_record(&hook("SessionEnd", "old"), 1).unwrap(); + mux.append_journal_ingress(&hook("SessionEnd", "old"), "test", "direct-hook-session-end") + .unwrap(); assert!( mux.report_agent( surface.id, From f5c5cf59f42722dfb4ea07c1386174c2ddb43da9 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:54:45 -0700 Subject: [PATCH 023/133] test(tui): cover paged roster folding --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index aca08983d7f4..a5c28761032e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22784,6 +22784,73 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn roster_fold_pages_to_a_target_beyond_the_first_journal_page() { + let root = std::env::temp_dir().join(format!( + "cmux-roster-multiple-pages-{}", + crate::workspace_registry::new_uuid_v4() + )); + let mux = Mux::open_persistent("roster-multiple-pages", SurfaceOptions::default(), &root) + .unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("terminal"); + let event = crate::JournalIngress { + producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), + manifest_version: crate::agent_hooks::AGENT_HOOK_MANIFEST_VERSION, + kind: "agent.state.changed".into(), + schema_version: 1, + occurred_at_ms: None, + subjects: vec![crate::JournalSubject { + kind: "terminal".into(), + id: terminal_id.to_string(), + }], + sensitivity: Some(crate::JournalSensitivity::Sensitive), + payload: serde_json::json!({ + "format": crate::agent_hooks::AGENT_HOOK_FORMAT, + "adapter": { + "id": crate::journal_reducers::SOCKET_REPORT_ADAPTER, + "version": 1, + }, + "native_event": crate::journal_reducers::SOCKET_REPORT_NATIVE_EVENT, + "normalized": { + "state": "working", + "source": "socket", + "source_session": "multiple-pages-test", + "updated_at_ms": "42", + }, + "native": {}, + }), + causation_id: None, + correlation_id: None, + }; + let validated = mux.journal_kernel.validate_ingress(&event).unwrap(); + let mut target = None; + for index in 0..513 { + target = Some( + mux.workspace_registry + .lock() + .unwrap() + .append_journal_ingress( + &event, + &validated, + "test", + &format!("roster-multiple-pages-{index}"), + ) + .unwrap(), + ); + } + let target = target.expect("target commit"); + + mux.fold_agent_roster(&event, &target); + let host = mux.agent_roster.lock().unwrap(); + assert_eq!(host.cursor, target.sequence); + assert!(host.roster.entries.contains_key(terminal_id.as_str())); + drop(host); + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn malformed_roster_snapshot_replays_from_the_journal_head() { let root = std::env::temp_dir() From 777331487646c8c279e4e0d6e623d85d46c6e492 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:56:00 -0700 Subject: [PATCH 024/133] fix(tui): page roster folds to committed target --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 54 +++++++++++++++--------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a5c28761032e..43e47dc8e2b7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5650,8 +5650,16 @@ impl Mux { } (host.cursor, host.roster.clone()) }; - let page = - match self.workspace_registry.lock().unwrap().session_journal_after(cursor, 512) { + let mut read_cursor = cursor; + let mut next_cursor = cursor; + let mut deltas = Vec::new(); + let reached_target = loop { + let page = match self + .workspace_registry + .lock() + .unwrap() + .session_journal_after(read_cursor, 512) + { Ok(page) => page, Err(error) => { // The durable event remains available for a later fold or @@ -5660,29 +5668,33 @@ impl Mux { return; } }; - if page.records.is_empty() { - return; - } - - let mut next_cursor = cursor; - let mut deltas = Vec::new(); - let mut reached_target = false; - for record in page.records { - if record.sequence <= next_cursor { - continue; + if page.records.is_empty() { + break false; } - // Do not consume a record committed after the callback's - // target. A later callback will fold that suffix. - if record.sequence > commit.sequence { - break; + + for record in page.records { + if record.sequence <= next_cursor { + continue; + } + // Do not consume a record committed after the callback's + // target. A later callback will fold that suffix. + if record.sequence > commit.sequence { + break; + } + deltas.extend(candidate.apply(&RosterEvent::from_record(&record))); + next_cursor = record.sequence; + if next_cursor == commit.sequence { + break; + } } - deltas.extend(candidate.apply(&RosterEvent::from_record(&record))); - next_cursor = record.sequence; if next_cursor == commit.sequence { - reached_target = true; - break; + break true; } - } + if next_cursor <= read_cursor { + break false; + } + read_cursor = next_cursor; + }; if !reached_target { // The writer acknowledged a commit that the reader cannot see // yet. Keep the cursor unchanged so another callback retries. From d97d36f17f1284b04f03414a332b7e23b4bdfe63 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:31:47 -0700 Subject: [PATCH 025/133] test(tui): cover terminal cleanup and stale roster recovery --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 70 +++++++++++++++++++ .../src/workspace_registry/session_journal.rs | 22 ++++++ 2 files changed, 92 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 43e47dc8e2b7..40400e33f5b7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24069,6 +24069,76 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn hosted_attachment_cleanup_kills_runtime_when_roster_purge_fails() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + assert!(!surface.is_dead(), "test terminal must start live"); + + mux.workspace_registry.lock().unwrap().set_journal_reducer_state_failure(true).unwrap(); + let cleanup = mux.fail_hosted_terminal_attachment( + &surface, + "terminal-topology-attach-failed", + "test-cleanup-failure", + ); + assert!(cleanup.is_err(), "the purge error must remain observable"); + assert!(surface.is_dead(), "cleanup must kill the runtime after purge failure"); + + mux.workspace_registry.lock().unwrap().set_journal_reducer_state_failure(false).unwrap(); + mux.shutdown(); + } + + #[test] + fn startup_reconciliation_removes_stale_ended_agent_projection() { + let root = std::env::temp_dir().join(format!( + "cmux-agent-stale-ended-projection-{}", + crate::workspace_registry::new_uuid_v4() + )); + let session = "agent-stale-ended-projection"; + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let hook = |event: &str| { + crate::agent_hooks::agent_hook_journal_ingress( + "claude", + event, + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"stale-ended-session"}), + ) + .unwrap() + }; + + mux.append_journal_ingress(&hook("SessionStart"), "test", "stale-ended-start").unwrap(); + assert_eq!(mux.list_agents(Some(surface.id), None).len(), 1); + + // Commit the end event to the journal only. This models a crash after + // the lifecycle event commit and before its compatibility projection. + let ended = hook("SessionEnd"); + let validated = mux.journal_kernel.validate_ingress(&ended).unwrap(); + mux.workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&ended, &validated, "test", "stale-ended-end") + .unwrap(); + assert_eq!( + mux.list_agents(Some(surface.id), None).len(), + 1, + "the test must leave a stale compatibility record before restart" + ); + + mux.shutdown(); + drop(mux); + let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let reopened_surface = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); + assert!( + reopened.list_agents(Some(reopened_surface), None).is_empty(), + "a reducer Remove must not resurrect an ended compatibility record" + ); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn failed_raw_agent_report_rolls_back_projection_memory_revision_and_event() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs index 4ea6c9672672..7584813bf256 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs @@ -937,6 +937,28 @@ impl WorkspaceRegistry { Ok(()) } + #[cfg(test)] + pub(crate) fn set_journal_reducer_state_failure(&self, enabled: bool) -> anyhow::Result<()> { + if enabled { + self.connection.execute_batch( + "CREATE TEMP TRIGGER cmux_test_fail_journal_reducer_state_insert + BEFORE INSERT ON meta + WHEN NEW.key LIKE 'journal_reducer.%' + BEGIN SELECT RAISE(ABORT, 'forced journal reducer state failure'); END; + CREATE TEMP TRIGGER cmux_test_fail_journal_reducer_state_update + BEFORE UPDATE OF value ON meta + WHEN OLD.key LIKE 'journal_reducer.%' + BEGIN SELECT RAISE(ABORT, 'forced journal reducer state failure'); END;", + )?; + } else { + self.connection.execute_batch( + "DROP TRIGGER IF EXISTS cmux_test_fail_journal_reducer_state_insert; + DROP TRIGGER IF EXISTS cmux_test_fail_journal_reducer_state_update;", + )?; + } + Ok(()) + } + /// The most recently started journal output stream for one terminal: /// its generation and the exclusive end offset of its journaled bytes. pub(crate) fn terminal_stream_latest( From 18c5be9b26b3b9c88e500ac9870a2df2c6c05207 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:50:08 -0700 Subject: [PATCH 026/133] fix(tui): finish terminal cleanup and reconcile roster removals --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 36 ++++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 40400e33f5b7..347a41601f19 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1094,6 +1094,11 @@ impl AgentSource { struct AgentRosterHost { roster: crate::journal_reducers::AgentRoster, cursor: u64, + /// A valid reducer snapshot or an observed agent journal event makes the + /// roster authoritative. Older sessions may only have the compatibility + /// projection, which must remain available until that durable source is + /// present. + authoritative: bool, } fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { @@ -1110,7 +1115,11 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { snapshot_valid = true; - AgentRosterHost { roster, cursor } + let authoritative = cursor > 0 + || !roster.entries.is_empty() + || !roster.hook_fences.is_empty() + || !roster.retired_terminals.is_empty(); + AgentRosterHost { roster, cursor, authoritative } } None => { eprintln!("cmux-tui: invalid agent roster snapshot; replaying journal"); @@ -1121,17 +1130,20 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result AgentRosterHost::default(), }; let initial_cursor = host.cursor; + let mut saw_agent_event = false; loop { let page = registry.session_journal_after(host.cursor, 512)?; if page.records.is_empty() { break; } for record in &page.records { + saw_agent_event |= record.producer.id == crate::agent_hooks::AGENT_HOOK_PRODUCER_ID; host.roster.apply(&RosterEvent::from_record(record)); host.cursor = host.cursor.max(record.sequence); } } - if host.cursor != initial_cursor || !snapshot_valid { + host.authoritative |= saw_agent_event; + if host.authoritative && (host.cursor != initial_cursor || !snapshot_valid) { registry.put_journal_reducer_state( AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, @@ -2435,6 +2447,18 @@ impl Mux { // legacy record cache in sync for compatibility with direct hook // projection paths and older callers that inspect it during startup. let mut agent_records = agent_records; + if agent_roster.authoritative { + let roster_terminal_ids = agent_roster + .roster + .entries + .keys() + .filter_map(|terminal_id| TerminalPublicId::parse(terminal_id).ok()) + .collect::>(); + // A reducer Remove means that the agent ended. Do not resurrect + // its old compatibility projection after a crash between the + // journal commit and projection update. + agent_records.retain(|terminal_id, _| roster_terminal_ids.contains(terminal_id)); + } for (terminal_id, entry) in &agent_roster.roster.entries { let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) else { continue }; let record = TerminalAgentRecord { @@ -9028,11 +9052,11 @@ impl Mux { for placement in removed { self.purge_surface_side_tables(placement.id); } - self.purge_terminal_runtime_side_tables(surface)?; + let purge_result = self.purge_terminal_runtime_side_tables(surface); if !surface.is_dead() { surface.kill(); } - return Ok(()); + return purge_result; }; self.persist_terminal_exit( &identity.terminal_id, @@ -9046,11 +9070,11 @@ impl Mux { for placement in removed { self.purge_surface_side_tables(placement.id); } - self.purge_terminal_runtime_side_tables(surface)?; + let purge_result = self.purge_terminal_runtime_side_tables(surface); if !surface.is_dead() { surface.kill(); } - Ok(()) + purge_result } pub(crate) fn terminate_discovered_terminal_host( From 3d1b369efbf975243f8d4cbf0ece7dc02a8e3c96 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:10:50 -0700 Subject: [PATCH 027/133] test(tui): cover roster fence retention and authority --- .../cmux-tui-core/src/journal_reducers.rs | 30 +++++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 86 ++++++++++++++++++- 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 047dbb5d6693..4105df5db768 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -601,4 +601,34 @@ mod tests { assert_eq!(deltas.len(), 1); assert!(!roster.is_retired("term_a")); } + + #[test] + fn terminal_fences_are_bounded() { + const EXPECTED_FENCE_LIMIT: usize = 1_024; + let payload = json!({}); + let mut live = AgentRoster::default(); + for sequence in 1..=EXPECTED_FENCE_LIMIT * 2 { + let subjects = terminal_subject("long_lived"); + live.apply(&hook_event( + sequence as u64, + if sequence % 2 == 0 { "agent.session.started" } else { "agent.turn.started" }, + &subjects, + &payload, + )); + } + assert_eq!(live.hook_fences.len(), 1); + + let mut retired = AgentRoster::default(); + for index in 0..(EXPECTED_FENCE_LIMIT * 2) { + let terminal_id = format!("retired_{index}"); + let subjects = terminal_subject(&terminal_id); + let sequence = index as u64 * 2 + 1; + retired.apply(&hook_event(sequence, "agent.turn.started", &subjects, &payload)); + retired.apply(&hook_event(sequence + 1, "agent.session.ended", &subjects, &payload)); + retired.retire_terminal(&terminal_id, sequence); + } + assert!(retired.hook_fences.is_empty()); + assert!(retired.retired_terminals.len() <= EXPECTED_FENCE_LIMIT); + assert_eq!(retired.retired_terminals.get("retired_2047"), Some(&4_095)); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 347a41601f19..122c3c43d0a8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22990,6 +22990,7 @@ mod tests { .join(format!("cmux-roster-echo-{}", crate::workspace_registry::new_uuid_v4())); let mux = Mux::open_persistent("roster-echo", SurfaceOptions::default(), &root).unwrap(); let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); let socket_echoes = || { mux.session_journal_after(0, 1024) .unwrap() @@ -23007,18 +23008,97 @@ mod tests { }; assert_eq!(socket_echoes(), 0); - mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) + let first = mux + .report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) .unwrap(); assert_eq!(socket_echoes(), 1); - mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) + std::thread::sleep(Duration::from_millis(5)); + let repeated = mux + .report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) .unwrap(); assert_eq!(socket_echoes(), 1); - mux.report_agent(surface.id, AgentState::Blocked, AgentSource::Socket, Some("poll".into())) + assert_eq!(repeated.updated_at_ms, first.updated_at_ms); + let durable_timestamp = first.updated_at_ms; + + mux.shutdown(); + drop(mux); + let reopened = + Mux::open_persistent("roster-echo", SurfaceOptions::default(), &root).unwrap(); + let reopened_surface = reopened + .resource_surface_for_terminal(&terminal_id) + .expect("reopened workspace terminal"); + assert_eq!( + reopened.list_agents(Some(reopened_surface), None)[0].updated_at_ms, + durable_timestamp + ); + let socket_echoes = || { + reopened + .session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count() + }; + assert_eq!(socket_echoes(), 1); + reopened + .report_agent( + reopened_surface, + AgentState::Blocked, + AgentSource::Socket, + Some("poll".into()), + ) .unwrap(); assert_eq!(socket_echoes(), 2); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn unsupported_hook_events_do_not_make_compatibility_projection_authoritative() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-unsupported-{}", crate::workspace_registry::new_uuid_v4())); + let session = "roster-unsupported"; + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Hook, + Some("compatibility-session".into()), + ) + .unwrap(); + + let child = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "SubagentStart", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"child-session"}), + ) + .unwrap(); + assert_eq!(child.kind, "agent.child.spawned"); + mux.append_journal_ingress(&child, "test", "unsupported-hook").unwrap(); + assert_eq!(mux.list_agents(Some(surface.id), None).len(), 1); mux.shutdown(); drop(mux); + let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let reopened_surface = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); + let records = reopened.list_agents(Some(reopened_surface), None); + assert_eq!(records.len(), 1); + assert_eq!(records[0].source, AgentSource::Hook); + assert_eq!(records[0].session.as_deref(), Some("compatibility-session")); + reopened.shutdown(); + drop(reopened); std::fs::remove_dir_all(root).unwrap(); } From a63e7afe815e187ca6590069aede33cc19806a0e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:33:42 -0700 Subject: [PATCH 028/133] fix(tui): bound and validate roster fences --- .../cmux-tui-core/src/journal_reducers.rs | 56 +++++++++++++++++-- cmux-tui/crates/cmux-tui-core/src/mux.rs | 53 +++++++++++++++--- 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 4105df5db768..3f5b1e12a562 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -16,7 +16,7 @@ //! echo event after its direct projection commit), so the roster never has //! a second writer to diverge from. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -28,8 +28,15 @@ use crate::{AgentSource, AgentState, JournalSubject}; pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; /// Bump to discard persisted snapshots and re-fold from the journal head. /// Version 2 added the agent adapter id to roster entries. Version 3 retains -/// ended hook fences after their live roster entries are removed. -pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 3; +/// ended hook fences after their live roster entries are removed. Version 4 +/// records whether a semantically valid roster event has been folded, so an +/// unrelated hook row cannot make the compatibility projection authoritative. +pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 4; + +/// Retirement tombstones are only needed while a terminal is still visible +/// to the durable topology. Keep a bounded safety net for a close/reconcile +/// race, then let topology reconciliation drop tombstones for gone terminals. +pub(crate) const MAX_RETIRED_TERMINAL_FENCES: usize = 1_024; /// The adapter id and native event the socket report path uses for its echo /// journal events. The echo carries the explicit state in `normalized`, so @@ -164,6 +171,11 @@ pub(crate) enum RosterDelta { #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub(crate) struct AgentRoster { pub(crate) entries: HashMap, + /// True only after a valid terminal-scoped roster event or an explicit + /// terminal retirement has been folded. This flag is persisted separately + /// from the cursor because unrelated producer rows can advance the latter. + #[serde(default)] + authoritative: bool, /// Hook generation watermarks remain after an ended session leaves the /// live roster. Socket echoes cannot recreate an ended entry, and delayed /// events from an old named session cannot cross into a newer lifecycle. @@ -215,6 +227,7 @@ impl AgentRoster { if self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) { return Vec::new(); } + self.authoritative = true; (state, source, session, None, updated_at_ms) } else { let Some(state) = state_for_hook_kind(event.kind) else { return Vec::new() }; @@ -252,6 +265,7 @@ impl AgentRoster { return Vec::new(); } } + self.authoritative = true; self.hook_fences.insert( terminal_id.to_string(), RosterFence { @@ -313,8 +327,12 @@ impl AgentRoster { /// tombstoned). Terminal lifecycle does not flow through `agent.*` /// events yet, so the host retires entries explicitly. pub(crate) fn retire_terminal(&mut self, terminal_id: &str, retired_at: u64) -> bool { + self.authoritative = true; let removed_entry = self.entries.remove(terminal_id).is_some(); - match self.retired_terminals.entry(terminal_id.to_string()) { + // A retired terminal is no longer allowed to receive socket echoes, + // and its ended hook fence is redundant with the retirement cursor. + let removed_fence = self.hook_fences.remove(terminal_id).is_some(); + let retired_changed = match self.retired_terminals.entry(terminal_id.to_string()) { std::collections::btree_map::Entry::Occupied(mut entry) => { let previous = *entry.get(); let effective = previous.max(retired_at); @@ -325,7 +343,37 @@ impl AgentRoster { entry.insert(retired_at); true } + }; + let pruned = self.prune_retired_terminals(); + removed_entry || removed_fence || retired_changed || pruned + } + + fn prune_retired_terminals(&mut self) -> bool { + let mut changed = false; + while self.retired_terminals.len() > MAX_RETIRED_TERMINAL_FENCES { + let oldest = self + .retired_terminals + .iter() + .min_by_key(|(terminal_id, sequence)| (**sequence, terminal_id.as_str())) + .map(|(terminal_id, _)| terminal_id.clone()); + let Some(oldest) = oldest else { break }; + self.retired_terminals.remove(&oldest); + changed = true; } + changed + } + + /// Remove fences for terminals no longer present in the durable resource + /// topology. This keeps snapshots proportional to live terminals and is + /// safe because direct reports cannot resolve a tombstoned terminal. + pub(crate) fn retain_live_terminals(&mut self, live_terminal_ids: &HashSet) { + self.entries.retain(|terminal_id, _| live_terminal_ids.contains(terminal_id)); + self.hook_fences.retain(|terminal_id, _| live_terminal_ids.contains(terminal_id)); + self.retired_terminals.retain(|terminal_id, _| live_terminal_ids.contains(terminal_id)); + } + + pub(crate) fn is_authoritative(&self) -> bool { + self.authoritative } pub(crate) fn is_retired(&self, terminal_id: &str) -> bool { diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 122c3c43d0a8..b598be390a4b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1115,10 +1115,7 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { snapshot_valid = true; - let authoritative = cursor > 0 - || !roster.entries.is_empty() - || !roster.hook_fences.is_empty() - || !roster.retired_terminals.is_empty(); + let authoritative = roster.is_authoritative(); AgentRosterHost { roster, cursor, authoritative } } None => { @@ -1130,20 +1127,26 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result AgentRosterHost::default(), }; let initial_cursor = host.cursor; - let mut saw_agent_event = false; loop { let page = registry.session_journal_after(host.cursor, 512)?; if page.records.is_empty() { break; } for record in &page.records { - saw_agent_event |= record.producer.id == crate::agent_hooks::AGENT_HOOK_PRODUCER_ID; host.roster.apply(&RosterEvent::from_record(record)); host.cursor = host.cursor.max(record.sequence); } } - host.authoritative |= saw_agent_event; - if host.authoritative && (host.cursor != initial_cursor || !snapshot_valid) { + let before_topology_prune = host.roster.snapshot(); + let live_terminal_ids = registry + .live_terminal_resource_ids()? + .into_iter() + .map(|(_, terminal_id)| terminal_id.to_string()) + .collect::>(); + host.roster.retain_live_terminals(&live_terminal_ids); + let topology_pruned = host.roster.snapshot() != before_topology_prune; + host.authoritative |= host.roster.is_authoritative(); + if host.authoritative && (host.cursor != initial_cursor || !snapshot_valid || topology_pruned) { registry.put_journal_reducer_state( AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, @@ -5724,7 +5727,28 @@ impl Mux { // yet. Keep the cursor unchanged so another callback retries. return; } + let live_terminal_ids = + match self.workspace_registry.lock().unwrap().live_terminal_resource_ids() { + Ok(terminals) => terminals + .into_iter() + .map(|(_, terminal_id)| terminal_id.to_string()) + .collect::>(), + Err(error) => { + eprintln!( + "cmux-tui: reading live terminals before roster prune failed: {error}" + ); + return; + } + }; + candidate.retain_live_terminals(&live_terminal_ids); + deltas.retain(|delta| match delta { + crate::journal_reducers::RosterDelta::Upsert { terminal_id, .. } + | crate::journal_reducers::RosterDelta::Remove { terminal_id } => { + live_terminal_ids.contains(terminal_id) + } + }); let snapshot = candidate.snapshot().to_string(); + let candidate_authoritative = candidate.is_authoritative(); if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, @@ -5743,6 +5767,7 @@ impl Mux { } host.roster = candidate; host.cursor = next_cursor; + host.authoritative |= candidate_authoritative; } for delta in deltas { self.apply_roster_delta_to_record_cache(delta); @@ -9612,8 +9637,18 @@ impl Mux { && source == AgentSource::Socket && !effective_hook_state.is_some_and(|state| state.ended) }); + let socket_report_unchanged = source == AgentSource::Socket + && records.get(&terminal_id).is_some_and(|existing| { + existing.source == AgentSource::Socket + && existing.state == agent_state + && existing.session.as_ref() == source_session.as_ref() + && existing.agent.as_ref() == agent_adapter.as_ref() + }); + // The echo reducer coalesces this semantic no-op. Keep its original + // timestamp too, so the compatibility projection and durable roster + // retain identical values across a restart. let record = match records.get(&terminal_id) { - Some(existing) if socket_report_ignored => existing.clone(), + Some(existing) if socket_report_ignored || socket_report_unchanged => existing.clone(), _ => TerminalAgentRecord { state: agent_state, source, From f3c0f71ff9d14e42853aa8d0354cfdf37fe16c2d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:52:00 -0700 Subject: [PATCH 029/133] fix(tui): reconcile roster state safely --- .../cmux-tui-core/src/journal_reducers.rs | 33 ++++++---- cmux-tui/crates/cmux-tui-core/src/mux.rs | 62 +++++++------------ cmux-tui/crates/cmux-tui-core/src/surface.rs | 8 +++ 3 files changed, 50 insertions(+), 53 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 3f5b1e12a562..967cf124ee22 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -16,7 +16,7 @@ //! echo event after its direct projection commit), so the roster never has //! a second writer to diverge from. -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -33,9 +33,9 @@ pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; /// unrelated hook row cannot make the compatibility projection authoritative. pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 4; -/// Retirement tombstones are only needed while a terminal is still visible -/// to the durable topology. Keep a bounded safety net for a close/reconcile -/// race, then let topology reconciliation drop tombstones for gone terminals. +/// Retirement tombstones protect delayed journal rows after a terminal leaves +/// the resource tree. Keep a bounded safety net. Durable startup reconciliation +/// removes entries for terminals that are confirmed gone. pub(crate) const MAX_RETIRED_TERMINAL_FENCES: usize = 1_024; /// The adapter id and native event the socket report path uses for its echo @@ -363,15 +363,6 @@ impl AgentRoster { changed } - /// Remove fences for terminals no longer present in the durable resource - /// topology. This keeps snapshots proportional to live terminals and is - /// safe because direct reports cannot resolve a tombstoned terminal. - pub(crate) fn retain_live_terminals(&mut self, live_terminal_ids: &HashSet) { - self.entries.retain(|terminal_id, _| live_terminal_ids.contains(terminal_id)); - self.hook_fences.retain(|terminal_id, _| live_terminal_ids.contains(terminal_id)); - self.retired_terminals.retain(|terminal_id, _| live_terminal_ids.contains(terminal_id)); - } - pub(crate) fn is_authoritative(&self) -> bool { self.authoritative } @@ -380,6 +371,22 @@ impl AgentRoster { self.retired_terminals.contains_key(terminal_id) } + /// Return every terminal represented by durable roster state. Startup + /// reconciliation uses this list to remove fences whose terminal resource + /// is confirmed gone, while preserving rows whose host mapping is still + /// being rebuilt. + pub(crate) fn fenced_terminal_ids(&self) -> Vec { + let mut ids = Vec::with_capacity( + self.entries.len() + self.hook_fences.len() + self.retired_terminals.len(), + ); + ids.extend(self.entries.keys().cloned()); + ids.extend(self.hook_fences.keys().cloned()); + ids.extend(self.retired_terminals.keys().cloned()); + ids.sort_unstable(); + ids.dedup(); + ids + } + pub(crate) fn has_ended_hook_fence(&self, terminal_id: &str) -> bool { self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b598be390a4b..7ee6402644fc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1137,16 +1137,8 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result>(); - host.roster.retain_live_terminals(&live_terminal_ids); - let topology_pruned = host.roster.snapshot() != before_topology_prune; host.authoritative |= host.roster.is_authoritative(); - if host.authoritative && (host.cursor != initial_cursor || !snapshot_valid || topology_pruned) { + if host.authoritative && (host.cursor != initial_cursor || !snapshot_valid) { registry.put_journal_reducer_state( AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, @@ -2676,8 +2668,7 @@ impl Mux { candidates.extend(self.agent_records.lock().unwrap().keys().cloned()); candidates.extend(self.agent_hook_fences.lock().unwrap().keys().cloned()); candidates.extend(self.terminal_notifications.lock().unwrap().keys().cloned()); - let roster_terminal_ids = - self.agent_roster.lock().unwrap().roster.entries.keys().cloned().collect::>(); + let roster_terminal_ids = self.agent_roster.lock().unwrap().roster.fenced_terminal_ids(); for terminal_id in roster_terminal_ids { if let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) { candidates.insert(terminal_id); @@ -2690,9 +2681,12 @@ impl Mux { let Some(host_id) = registry.terminal_host_id(&terminal_id)? else { return Ok(true); }; + // A resource row can outlive its host record while a + // reconnect is still materializing. Treat that gap as + // unknown, not as proof that the terminal was retired. Ok(registry .terminal_record(&host_id)? - .is_none_or(|terminal| terminal.lifecycle == TerminalLifecycle::Tombstoned)) + .is_some_and(|terminal| terminal.lifecycle == TerminalLifecycle::Tombstoned)) })(); match retired { Ok(true) => { @@ -5727,26 +5721,6 @@ impl Mux { // yet. Keep the cursor unchanged so another callback retries. return; } - let live_terminal_ids = - match self.workspace_registry.lock().unwrap().live_terminal_resource_ids() { - Ok(terminals) => terminals - .into_iter() - .map(|(_, terminal_id)| terminal_id.to_string()) - .collect::>(), - Err(error) => { - eprintln!( - "cmux-tui: reading live terminals before roster prune failed: {error}" - ); - return; - } - }; - candidate.retain_live_terminals(&live_terminal_ids); - deltas.retain(|delta| match delta { - crate::journal_reducers::RosterDelta::Upsert { terminal_id, .. } - | crate::journal_reducers::RosterDelta::Remove { terminal_id } => { - live_terminal_ids.contains(terminal_id) - } - }); let snapshot = candidate.snapshot().to_string(); let candidate_authoritative = candidate.is_authoritative(); if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( @@ -5813,6 +5787,7 @@ impl Mux { state: AgentState, source: AgentSource, session: Option<&str>, + updated_at_ms: u64, ) -> Option { let (cursor, previous) = { let host = self.agent_roster.lock().unwrap(); @@ -5828,10 +5803,11 @@ impl Mux { (host.cursor, previous) }; let unchanged = previous.as_ref().is_some_and( - |(previous_state, previous_source, previous_session, _, _)| { + |(previous_state, previous_source, previous_session, _, previous_updated_at_ms)| { previous_state == state.as_str() && previous_source == source.as_str() && previous_session.as_deref() == session + && *previous_updated_at_ms == updated_at_ms }, ); if unchanged { @@ -9762,6 +9738,7 @@ impl Mux { agent.state, agent.source, agent.session.as_deref(), + agent.updated_at_ms, ) { self.append_agent_report_echo( @@ -22962,7 +22939,9 @@ mod tests { ) .unwrap(); let restored = restore_agent_roster(®istry).unwrap(); - assert_eq!(restored.cursor, sequence); + let journal_head = registry.session_journal_after(0, 1).unwrap().head_sequence; + assert_eq!(restored.cursor, journal_head); + assert!(journal_head >= sequence); assert_eq!(restored.roster.entries[terminal_id.as_str()].agent.as_deref(), Some("claude")); std::fs::remove_dir_all(root).unwrap(); } @@ -23051,9 +23030,9 @@ mod tests { let repeated = mux .report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) .unwrap(); - assert_eq!(socket_echoes(), 1); - assert_eq!(repeated.updated_at_ms, first.updated_at_ms); - let durable_timestamp = first.updated_at_ms; + assert_eq!(socket_echoes(), 2, "a changed freshness timestamp is durable state"); + assert!(repeated.updated_at_ms >= first.updated_at_ms); + let durable_timestamp = repeated.updated_at_ms; mux.shutdown(); drop(mux); @@ -23082,7 +23061,7 @@ mod tests { }) .count() }; - assert_eq!(socket_echoes(), 1); + assert_eq!(socket_echoes(), 2); reopened .report_agent( reopened_surface, @@ -23091,7 +23070,7 @@ mod tests { Some("poll".into()), ) .unwrap(); - assert_eq!(socket_echoes(), 2); + assert_eq!(socket_echoes(), 3); reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); @@ -24221,7 +24200,10 @@ mod tests { "test-cleanup-failure", ); assert!(cleanup.is_err(), "the purge error must remain observable"); - assert!(surface.is_dead(), "cleanup must kill the runtime after purge failure"); + assert!( + surface.kill_requested_for_test(), + "cleanup must request runtime termination after purge failure" + ); mux.workspace_registry.lock().unwrap().set_journal_reducer_state_failure(false).unwrap(); mux.shutdown(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 1ac55ea399a1..229bbcfabe00 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -5405,6 +5405,14 @@ impl Surface { } } + /// Test-only probe for the authoritative detach request. A local test + /// killer does not have a child process to reap, so `is_dead()` cannot + /// observe the request immediately. + #[cfg(test)] + pub(crate) fn kill_requested_for_test(&self) -> bool { + self.as_pty().is_some_and(|pty| pty.owner_detaching.load(Ordering::Acquire)) + } + /// Clear the coalesced output flag; returns whether output was pending. pub fn take_dirty(&self) -> bool { match self { From 9f3b96cf65b45135d3e282c735bb24f9a4fbe6e0 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:59:56 -0700 Subject: [PATCH 030/133] fix(tui): persist socket report freshness --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7ee6402644fc..66f1c5b80228 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9613,18 +9613,11 @@ impl Mux { && source == AgentSource::Socket && !effective_hook_state.is_some_and(|state| state.ended) }); - let socket_report_unchanged = source == AgentSource::Socket - && records.get(&terminal_id).is_some_and(|existing| { - existing.source == AgentSource::Socket - && existing.state == agent_state - && existing.session.as_ref() == source_session.as_ref() - && existing.agent.as_ref() == agent_adapter.as_ref() - }); - // The echo reducer coalesces this semantic no-op. Keep its original - // timestamp too, so the compatibility projection and durable roster - // retain identical values across a restart. + // Hook-owned state remains a semantic no-op for a socket report. A + // socket-owned report still refreshes its durable timestamp; the + // echo key below coalesces only when that complete payload is equal. let record = match records.get(&terminal_id) { - Some(existing) if socket_report_ignored || socket_report_unchanged => existing.clone(), + Some(existing) if socket_report_ignored => existing.clone(), _ => TerminalAgentRecord { state: agent_state, source, From 324f2dbfd823c23b6abf9bba3650a664c099401d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:15:01 -0700 Subject: [PATCH 031/133] test(tui): scope roster removal authority and coalesced freshness --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 65 ++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 66f1c5b80228..bf7618e834aa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23023,8 +23023,8 @@ mod tests { let repeated = mux .report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) .unwrap(); - assert_eq!(socket_echoes(), 2, "a changed freshness timestamp is durable state"); - assert!(repeated.updated_at_ms >= first.updated_at_ms); + assert_eq!(socket_echoes(), 1, "identical socket polls are journal-coalesced"); + assert!(repeated.updated_at_ms > first.updated_at_ms); let durable_timestamp = repeated.updated_at_ms; mux.shutdown(); @@ -23054,7 +23054,7 @@ mod tests { }) .count() }; - assert_eq!(socket_echoes(), 2); + assert_eq!(socket_echoes(), 1); reopened .report_agent( reopened_surface, @@ -23063,7 +23063,7 @@ mod tests { Some("poll".into()), ) .unwrap(); - assert_eq!(socket_echoes(), 3); + assert_eq!(socket_echoes(), 2); reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); @@ -23109,6 +23109,63 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn roster_removal_authority_is_scoped_to_the_removed_terminal() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-authority-scope-{}", crate::workspace_registry::new_uuid_v4())); + let session = "roster-authority-scope"; + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let ended_surface = mux.new_workspace(None, None).unwrap(); + let compatibility_surface = mux.new_workspace(None, None).unwrap(); + let ended_terminal = ended_surface.terminal_public_id().cloned().expect("ended terminal"); + let compatibility_terminal = compatibility_surface + .terminal_public_id() + .cloned() + .expect("compatibility terminal"); + + let start = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "SessionStart", + Some(&ended_terminal.to_string()), + serde_json::json!({"session_id":"ended-session"}), + ) + .unwrap(); + mux.append_journal_ingress(&start, "test", "authority-scope-start").unwrap(); + let end = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "SessionEnd", + Some(&ended_terminal.to_string()), + serde_json::json!({"session_id":"ended-session"}), + ) + .unwrap(); + mux.append_journal_ingress(&end, "test", "authority-scope-end").unwrap(); + + // Direct hook reports remain in the compatibility projection until + // their own terminal has a durable roster removal fence. + mux.report_agent( + compatibility_surface.id, + AgentState::Working, + AgentSource::Hook, + Some("compatibility-session".into()), + ) + .unwrap(); + mux.shutdown(); + drop(mux); + + let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + assert!(reopened.list_agents(Some(ended_surface.id), None).is_empty()); + let compatibility_surface = reopened + .resource_surface_for_terminal(&compatibility_terminal) + .expect("compatibility terminal restored"); + let records = reopened.list_agents(Some(compatibility_surface), None); + assert_eq!(records.len(), 1); + assert_eq!(records[0].source, AgentSource::Hook); + assert_eq!(records[0].session.as_deref(), Some("compatibility-session")); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn stale_same_terminal_hook_sequence_cannot_overwrite_newer_state() { let mux = test_mux(); From de1931e52c8f13fe62944d40bcb551bbdf512b36 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:17:08 -0700 Subject: [PATCH 032/133] fix(tui): scope roster authority and coalesce socket echoes --- .../cmux-tui-core/src/journal_reducers.rs | 24 +++--- cmux-tui/crates/cmux-tui-core/src/mux.rs | 78 +++++++++---------- 2 files changed, 47 insertions(+), 55 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 967cf124ee22..e31a9c9c7014 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -29,9 +29,9 @@ pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; /// Bump to discard persisted snapshots and re-fold from the journal head. /// Version 2 added the agent adapter id to roster entries. Version 3 retains /// ended hook fences after their live roster entries are removed. Version 4 -/// records whether a semantically valid roster event has been folded, so an -/// unrelated hook row cannot make the compatibility projection authoritative. -pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 4; +/// added a global authority bit. Version 5 removes that global bit and scopes +/// compatibility cleanup to each terminal's durable removal fence. +pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 5; /// Retirement tombstones protect delayed journal rows after a terminal leaves /// the resource tree. Keep a bounded safety net. Durable startup reconciliation @@ -171,11 +171,6 @@ pub(crate) enum RosterDelta { #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub(crate) struct AgentRoster { pub(crate) entries: HashMap, - /// True only after a valid terminal-scoped roster event or an explicit - /// terminal retirement has been folded. This flag is persisted separately - /// from the cursor because unrelated producer rows can advance the latter. - #[serde(default)] - authoritative: bool, /// Hook generation watermarks remain after an ended session leaves the /// live roster. Socket echoes cannot recreate an ended entry, and delayed /// events from an old named session cannot cross into a newer lifecycle. @@ -227,7 +222,6 @@ impl AgentRoster { if self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) { return Vec::new(); } - self.authoritative = true; (state, source, session, None, updated_at_ms) } else { let Some(state) = state_for_hook_kind(event.kind) else { return Vec::new() }; @@ -265,7 +259,6 @@ impl AgentRoster { return Vec::new(); } } - self.authoritative = true; self.hook_fences.insert( terminal_id.to_string(), RosterFence { @@ -327,7 +320,6 @@ impl AgentRoster { /// tombstoned). Terminal lifecycle does not flow through `agent.*` /// events yet, so the host retires entries explicitly. pub(crate) fn retire_terminal(&mut self, terminal_id: &str, retired_at: u64) -> bool { - self.authoritative = true; let removed_entry = self.entries.remove(terminal_id).is_some(); // A retired terminal is no longer allowed to receive socket echoes, // and its ended hook fence is redundant with the retirement cursor. @@ -363,12 +355,16 @@ impl AgentRoster { changed } - pub(crate) fn is_authoritative(&self) -> bool { - self.authoritative + pub(crate) fn is_retired(&self, terminal_id: &str) -> bool { + self.retired_terminals.contains_key(terminal_id) } - pub(crate) fn is_retired(&self, terminal_id: &str) -> bool { + /// Return true when this terminal has a durable roster decision that + /// removes its compatibility record. The decision is terminal-local, so + /// an event for one terminal cannot discard a different terminal's record. + pub(crate) fn has_terminal_removal_fence(&self, terminal_id: &str) -> bool { self.retired_terminals.contains_key(terminal_id) + || self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) } /// Return every terminal represented by durable roster state. Startup diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bf7618e834aa..263ab257a474 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1094,11 +1094,6 @@ impl AgentSource { struct AgentRosterHost { roster: crate::journal_reducers::AgentRoster, cursor: u64, - /// A valid reducer snapshot or an observed agent journal event makes the - /// roster authoritative. Older sessions may only have the compatibility - /// projection, which must remain available until that durable source is - /// present. - authoritative: bool, } fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { @@ -1115,8 +1110,7 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { snapshot_valid = true; - let authoritative = roster.is_authoritative(); - AgentRosterHost { roster, cursor, authoritative } + AgentRosterHost { roster, cursor } } None => { eprintln!("cmux-tui: invalid agent roster snapshot; replaying journal"); @@ -1137,8 +1131,7 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result>(); - // A reducer Remove means that the agent ended. Do not resurrect - // its old compatibility projection after a crash between the - // journal commit and projection update. - agent_records.retain(|terminal_id, _| roster_terminal_ids.contains(terminal_id)); - } + // A reducer Remove means that this terminal ended. Do not resurrect + // its old compatibility projection after a crash between the journal + // commit and projection update. Keep records for other terminals that + // have no roster event yet, because direct hook reports still use the + // compatibility path. + agent_records.retain(|terminal_id, _| { + !agent_roster.roster.has_terminal_removal_fence(terminal_id.as_str()) + }); for (terminal_id, entry) in &agent_roster.roster.entries { let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) else { continue }; let record = TerminalAgentRecord { @@ -2463,7 +2452,20 @@ impl Mux { agent: entry.agent.clone(), updated_at_ms: entry.updated_at_ms, }; - agent_records.insert(terminal_id, record); + // A repeated socket poll refreshes the compatibility projection + // without appending a journal echo. On restart, retain that newer + // timestamp when the roster snapshot still has the older echo. + let keep_newer_socket_record = + agent_records.get(&terminal_id).is_some_and(|existing| { + existing.source == AgentSource::Socket + && record.source == AgentSource::Socket + && existing.state == record.state + && existing.session == record.session + && existing.updated_at_ms > record.updated_at_ms + }); + if !keep_newer_socket_record { + agent_records.insert(terminal_id, record); + } } let journal_producers = registry.journal_producer_manifests()?; let session_public_id = registry.session_id().clone(); @@ -5722,7 +5724,6 @@ impl Mux { return; } let snapshot = candidate.snapshot().to_string(); - let candidate_authoritative = candidate.is_authoritative(); if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, @@ -5741,7 +5742,6 @@ impl Mux { } host.roster = candidate; host.cursor = next_cursor; - host.authoritative |= candidate_authoritative; } for delta in deltas { self.apply_roster_delta_to_record_cache(delta); @@ -5778,16 +5778,15 @@ impl Mux { } /// Return a stable echo receipt for a semantic socket-state transition. - /// Repeated polls of the same state do not append another journal row; - /// the prior reducer state and cursor distinguish a later transition back - /// to that state from a duplicate poll. + /// Repeated polls of the same state do not append another journal row. + /// Their fresh timestamp remains in the direct compatibility projection; + /// only semantic changes need a reducer event. fn agent_report_echo_key( &self, terminal_id: &TerminalPublicId, state: AgentState, source: AgentSource, session: Option<&str>, - updated_at_ms: u64, ) -> Option { let (cursor, previous) = { let host = self.agent_roster.lock().unwrap(); @@ -5797,29 +5796,27 @@ impl Mux { entry.source.clone(), entry.session.clone(), entry.agent.clone(), - entry.updated_at_ms, ) }); (host.cursor, previous) }; let unchanged = previous.as_ref().is_some_and( - |(previous_state, previous_source, previous_session, _, previous_updated_at_ms)| { + |(previous_state, previous_source, previous_session, _)| { previous_state == state.as_str() && previous_source == source.as_str() && previous_session.as_deref() == session - && *previous_updated_at_ms == updated_at_ms }, ); if unchanged { return None; } let previous = previous - .map(|(state, source, session, agent, updated_at_ms)| { - format!("{state}|{source}|{:?}|{:?}|{updated_at_ms}", session, agent) + .map(|(state, source, session, agent)| { + format!("{state}|{source}|{:?}|{:?}", session, agent) }) .unwrap_or_default(); let material = format!( - "agent-report-echo-v1|{terminal_id}|{}|{}|{:?}|{cursor}|{previous}", + "agent-report-echo-v2|{terminal_id}|{}|{}|{:?}|{cursor}|{previous}", state.as_str(), source.as_str(), session, @@ -9731,7 +9728,6 @@ impl Mux { agent.state, agent.source, agent.session.as_deref(), - agent.updated_at_ms, ) { self.append_agent_report_echo( @@ -23111,17 +23107,17 @@ mod tests { #[test] fn roster_removal_authority_is_scoped_to_the_removed_terminal() { - let root = std::env::temp_dir() - .join(format!("cmux-roster-authority-scope-{}", crate::workspace_registry::new_uuid_v4())); + let root = std::env::temp_dir().join(format!( + "cmux-roster-authority-scope-{}", + crate::workspace_registry::new_uuid_v4() + )); let session = "roster-authority-scope"; let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); let ended_surface = mux.new_workspace(None, None).unwrap(); let compatibility_surface = mux.new_workspace(None, None).unwrap(); let ended_terminal = ended_surface.terminal_public_id().cloned().expect("ended terminal"); - let compatibility_terminal = compatibility_surface - .terminal_public_id() - .cloned() - .expect("compatibility terminal"); + let compatibility_terminal = + compatibility_surface.terminal_public_id().cloned().expect("compatibility terminal"); let start = crate::agent_hooks::agent_hook_journal_ingress( "claude", From 6ec40093ad0c2d97e3b1b7ac4ee699fa4544c171 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 02:23:02 -0700 Subject: [PATCH 033/133] fix: preserve reconnecting agent state during cleanup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 75 +++++++++++++++++++++--- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 263ab257a474..283b3d113edf 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2678,17 +2678,24 @@ impl Mux { } for terminal_id in candidates { + // A missing host mapping is ambiguous during startup: the + // resource row can be visible before its host record is + // materialized. Only a durable terminal retirement tombstone + // makes that absence proof that cleanup is safe. + let explicitly_retired = + self.agent_roster.lock().unwrap().roster.is_retired(terminal_id.as_str()); let retired: anyhow::Result = (|| { let registry = self.workspace_registry.lock().unwrap(); let Some(host_id) = registry.terminal_host_id(&terminal_id)? else { - return Ok(true); + return Ok(explicitly_retired); }; // A resource row can outlive its host record while a // reconnect is still materializing. Treat that gap as // unknown, not as proof that the terminal was retired. Ok(registry .terminal_record(&host_id)? - .is_some_and(|terminal| terminal.lifecycle == TerminalLifecycle::Tombstoned)) + .is_some_and(|terminal| terminal.lifecycle == TerminalLifecycle::Tombstoned) + || explicitly_retired) })(); match retired { Ok(true) => { @@ -9813,12 +9820,13 @@ impl Mux { } fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) -> anyhow::Result<()> { - if let Some(terminal_id) = runtime.terminal_public_id() { - // Persist the terminal tombstone before clearing any runtime cache. - // A failed journal read or snapshot write must leave all side tables - // available for a later retry. - self.purge_terminal_side_tables(terminal_id)?; - } + // Keep the durable error observable, but always clear the in-memory + // runtime reservations in this call. Close and rollback paths may + // intentionally ignore the durable error after they have killed the + // process, so leaving these entries behind would leak one per failure. + let purge_result = runtime + .terminal_public_id() + .map_or(Ok(()), |terminal_id| self.purge_terminal_side_tables(terminal_id)); if let Some(runtime_id) = runtime.terminal_runtime_id() { self.reserved_in_process_terminals.lock().unwrap().remove(&runtime_id); let _cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); @@ -9832,7 +9840,7 @@ impl Mux { } } } - Ok(()) + purge_result } pub fn list_agents( @@ -24233,6 +24241,29 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn startup_reconciliation_preserves_agent_when_host_mapping_is_missing() { + let mux = test_mux(); + let terminal_id = TerminalPublicId::parse("term_000000000000000000000000000000fe").unwrap(); + mux.agent_records.lock().unwrap().insert( + terminal_id.clone(), + TerminalAgentRecord { + state: AgentState::Working, + source: AgentSource::Hook, + session: Some("reconnect-gap".into()), + agent: Some("claude".into()), + updated_at_ms: 1, + }, + ); + + // No resource row means the host mapping is not available yet. It is + // not proof that this compatibility record belongs to a retired + // terminal, so reconciliation must leave it for a later retry. + mux.reconcile_retired_terminal_side_tables(); + assert!(mux.agent_records.lock().unwrap().contains_key(&terminal_id)); + mux.shutdown(); + } + #[test] fn hosted_attachment_cleanup_kills_runtime_when_roster_purge_fails() { let mux = test_mux(); @@ -24255,6 +24286,32 @@ mod tests { mux.shutdown(); } + #[test] + fn runtime_side_tables_are_cleared_when_roster_purge_fails() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let runtime_id = surface.terminal_runtime_id().unwrap_or(surface.id); + let identity = mux + .resource_terminal_host_identity(&surface) + .expect("test terminal has a host identity"); + mux.reserved_in_process_terminals.lock().unwrap().insert(runtime_id, identity); + *mux.pending_cell_pixels.lock().unwrap() = Some(PendingCellPixelUpdate { + generation: 1, + target: (9, 18), + failures: HashSet::from([runtime_id]), + use_for_creation: true, + }); + + mux.workspace_registry.lock().unwrap().set_journal_reducer_state_failure(true).unwrap(); + let result = mux.purge_terminal_runtime_side_tables(&surface); + assert!(result.is_err(), "the durable roster failure remains observable"); + assert!(!mux.reserved_in_process_terminals.lock().unwrap().contains_key(&runtime_id)); + assert!(mux.pending_cell_pixels.lock().unwrap().is_none()); + + mux.workspace_registry.lock().unwrap().set_journal_reducer_state_failure(false).unwrap(); + mux.shutdown(); + } + #[test] fn startup_reconciliation_removes_stale_ended_agent_projection() { let root = std::env::temp_dir().join(format!( From 7c2ce06cd02152181957b07ac728fbf7a37f0b7a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:35:19 -0700 Subject: [PATCH 034/133] fix(tui): fail closed on corrupt roster metadata --- .../cmux-tui-core/src/journal_reducers.rs | 30 ++++++++++++++++++- .../src/workspace_registry/session_journal.rs | 25 ++++++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index e31a9c9c7014..31c405bc2a4d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -150,7 +150,9 @@ impl RosterEntry { } pub(crate) fn agent_source(&self) -> AgentSource { - agent_source_from_str(&self.source).unwrap_or(AgentSource::Hook) + // Unknown persisted values are untrusted. Treat them as detected + // state so they never gain hook authority over a socket report. + agent_source_from_str(&self.source).unwrap_or(AgentSource::Detected) } } @@ -611,6 +613,32 @@ mod tests { assert!(!roster.entries.is_empty()); } + #[test] + fn unknown_snapshot_source_fails_closed_without_hook_authority() { + let entry = RosterEntry { + state: "working".into(), + source: "future-source".into(), + session: Some("future-session".into()), + agent: Some("future-agent".into()), + updated_at_ms: 1, + }; + assert_eq!(entry.agent_source(), AgentSource::Detected); + + let subjects = terminal_subject("term_a"); + let socket_payload = json!({ + "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, + "normalized": {"state": "idle", "source_session": "socket"} + }); + let mut roster = AgentRoster { + entries: HashMap::from([(String::from("term_a"), entry)]), + ..AgentRoster::default() + }; + let deltas = + roster.apply(&hook_event(2, "agent.state.changed", &subjects, &socket_payload)); + assert_eq!(deltas.len(), 1, "socket state must be allowed to replace an unknown source"); + assert_eq!(roster.entries["term_a"].source, "socket"); + } + #[test] fn retired_terminal_rejects_late_events_until_a_new_session_starts() { let subjects = terminal_subject("term_a"); diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs index 7584813bf256..c7eda0092aac 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs @@ -902,8 +902,14 @@ impl WorkspaceRegistry { ) .optional()?; let Some(raw) = raw else { return Ok(None) }; - let value: Value = serde_json::from_str(&raw) - .with_context(|| format!("journal reducer state for {reducer_id} is not JSON"))?; + let value: Value = match serde_json::from_str(&raw) { + Ok(value) => value, + Err(_) => { + // Reducer metadata is a rebuildable cache. Ignore a corrupt + // wrapper and let the caller replay the retained journal. + return Ok(None); + } + }; let version = value.get("version").and_then(Value::as_u64).unwrap_or(0) as u32; let cursor = value .get("cursor") @@ -1899,6 +1905,21 @@ pub(crate) fn unix_epoch_ms() -> anyhow::Result { mod tests { use super::*; + #[test] + fn malformed_reducer_metadata_is_treated_as_absent() { + let mut registry = WorkspaceRegistry::in_memory("reducer-metadata").unwrap(); + registry + .connection + .execute( + "INSERT INTO meta(key, value) VALUES(?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params!["journal_reducer.test", "{malformed"], + ) + .unwrap(); + + assert_eq!(registry.journal_reducer_state("test").unwrap(), None); + } + #[test] fn resource_record_is_typed_scoped_and_append_only() { let mut registry = WorkspaceRegistry::in_memory("journal").unwrap(); From 2425bb7177b5bfce24c7acde56230eacdad035d0 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:50:51 -0700 Subject: [PATCH 035/133] chore(tui): drop obsolete CDP context import --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index b4226156e8a1..9b87e453192c 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -10,7 +10,6 @@ use std::sync::mpsc::{ use std::sync::{Arc, Condvar, Mutex, Weak}; use std::time::{Duration, Instant}; -use anyhow::Context; use base64::Engine; use serde_json::{Value, json}; use tungstenite::client::IntoClientRequest; From 8b58228c753e0f734dfcf50e16a2f9e6eb866694 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:08:22 -0700 Subject: [PATCH 036/133] fix(tui): fold agent roster incrementally and reject direct hooks --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 145 +++++++++--------- .../src/workspace_registry/session_journal.rs | 12 ++ 2 files changed, 83 insertions(+), 74 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 283b3d113edf..c863e09b68c5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5662,98 +5662,88 @@ impl Mux { ingress: &crate::JournalIngress, commit: &crate::JournalAppendCommit, ) { - use crate::journal_reducers::{ - AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, RosterEvent, - }; + use crate::journal_reducers::RosterEvent; if ingress.producer_id != crate::agent_hooks::AGENT_HOOK_PRODUCER_ID { return; } // Journal callbacks can complete out of order. Serialize folds and // replay the contiguous committed prefix, so a later sequence never // advances the cursor past an earlier event that has not been folded. + // Apply directly to the in-memory reducer. The journal is the source + // of truth, so a checkpoint write may lag safely across a crash and + // the next startup replays only the uncheckpointed tail. Cloning the + // complete roster for every poll made callback cost grow with the + // number of terminals. let _fold = self.agent_roster_fold.lock().unwrap(); + let mut read_cursor = self.agent_roster.lock().unwrap().cursor; + if commit.sequence <= read_cursor { + return; + } loop { - let (cursor, mut candidate) = { - let host = self.agent_roster.lock().unwrap(); - if commit.sequence <= host.cursor { + let page = match self.workspace_registry.lock().unwrap().session_journal_after_subjects( + read_cursor, + 512, + &[crate::JournalSubject { + kind: "producer".into(), + id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), + }], + ) { + Ok(page) => page, + Err(error) => { + eprintln!("cmux-tui: reading committed agent events back failed: {error}"); return; } - (host.cursor, host.roster.clone()) }; - let mut read_cursor = cursor; - let mut next_cursor = cursor; - let mut deltas = Vec::new(); - let reached_target = loop { - let page = match self - .workspace_registry - .lock() - .unwrap() - .session_journal_after(read_cursor, 512) - { - Ok(page) => page, - Err(error) => { - // The durable event remains available for a later fold or - // restart. Leave the cursor unchanged when the read fails. - eprintln!("cmux-tui: reading committed agent events back failed: {error}"); - return; - } - }; - if page.records.is_empty() { - break false; + let scanned_through = page.scanned_through.min(commit.sequence); + for record in page.records { + if record.sequence <= read_cursor { + continue; } - - for record in page.records { - if record.sequence <= next_cursor { - continue; - } - // Do not consume a record committed after the callback's - // target. A later callback will fold that suffix. - if record.sequence > commit.sequence { - break; - } - deltas.extend(candidate.apply(&RosterEvent::from_record(&record))); - next_cursor = record.sequence; - if next_cursor == commit.sequence { - break; - } + if record.sequence > commit.sequence { + return; } - if next_cursor == commit.sequence { - break true; + let deltas = { + let mut host = self.agent_roster.lock().unwrap(); + let deltas = host.roster.apply(&RosterEvent::from_record(&record)); + host.cursor = record.sequence; + deltas + }; + for delta in deltas { + self.apply_roster_delta_to_record_cache(delta); } - if next_cursor <= read_cursor { - break false; + read_cursor = record.sequence; + // Checkpoint at a bounded journal interval. The in-memory + // reducer advances for every event, while durable snapshots + // lag by at most this interval and are rebuilt from the + // journal tail after restart. + if read_cursor % 64 == 0 { + let (cursor, snapshot) = { + let host = self.agent_roster.lock().unwrap(); + (host.cursor, host.roster.snapshot().to_string()) + }; + if let Err(error) = + self.workspace_registry.lock().unwrap().put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ) + { + eprintln!( + "cmux-tui: persisting the agent roster checkpoint failed: {error}" + ); + } } - read_cursor = next_cursor; - }; - if !reached_target { - // The writer acknowledged a commit that the reader cannot see - // yet. Keep the cursor unchanged so another callback retries. - return; - } - let snapshot = candidate.snapshot().to_string(); - if let Err(error) = self.workspace_registry.lock().unwrap().put_journal_reducer_state( - AGENT_ROSTER_REDUCER_ID, - AGENT_ROSTER_REDUCER_VERSION, - next_cursor, - &snapshot, - ) { - eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); - return; - } - { - let mut host = self.agent_roster.lock().unwrap(); - if host.cursor != cursor { - // The fold gate should make this impossible. Preserve a - // newer host state if a future writer violates that rule. + if read_cursor == commit.sequence { return; } - host.roster = candidate; - host.cursor = next_cursor; } - for delta in deltas { - self.apply_roster_delta_to_record_cache(delta); - } - if commit.sequence <= next_cursor { + // The filtered query scanned through this watermark. Advance + // over unrelated records, but never beyond the callback target. + if scanned_through > read_cursor { + read_cursor = scanned_through; + self.agent_roster.lock().unwrap().cursor = read_cursor; + } else { return; } } @@ -9489,6 +9479,13 @@ impl Mux { origin: AgentReportOrigin, agent_adapter: Option, ) -> anyhow::Result<(ResourcePatchCommit, Option)> { + // Hook state is accepted only through the validated journal ingress. + // A direct Hook report has no durable event to replay, so allowing it + // would let a restart overwrite the journal-derived projection. + anyhow::ensure!( + !(source == AgentSource::Hook && origin == AgentReportOrigin::Direct), + "direct Hook reports require a validated journal event" + ); // Hook replay already owns this guard to serialize sequence checks and // projection commits. Other report sources acquire it before the // registry/state locks, so all paths use one lock order. diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs index c7eda0092aac..e398388ca650 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/session_journal.rs @@ -886,6 +886,18 @@ impl WorkspaceRegistry { query_session_journal_after(&self.connection, sequence, limit) } + /// Read only records belonging to one producer while retaining the + /// journal scan watermark. Reducers can skip unrelated high-volume rows + /// without losing sequence ordering. + pub(crate) fn session_journal_after_subjects( + &self, + sequence: u64, + limit: usize, + subjects: &[JournalSubject], + ) -> anyhow::Result { + query_session_journal_after_subjects(&self.connection, sequence, limit, subjects) + } + /// Persisted fold position of one journal reducer: (version, cursor, /// snapshot). A version mismatch on load discards the snapshot so the /// reducer re-folds from the journal head. From 1c902627b71f5485958635aa8211883b00a0e778 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:09:54 -0700 Subject: [PATCH 037/133] fix(tui): journal socket reports before projection --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 91 +++++++++++++++++------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c863e09b68c5..3481a66ba39e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5838,7 +5838,7 @@ impl Mux { session: Option<&str>, updated_at_ms: u64, idempotency_key: &str, - ) { + ) -> anyhow::Result { use crate::journal_reducers::{SOCKET_REPORT_ADAPTER, SOCKET_REPORT_NATIVE_EVENT}; let ingress = crate::JournalIngress { producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), @@ -5866,9 +5866,7 @@ impl Mux { causation_id: None, correlation_id: None, }; - if let Err(error) = self.append_journal_ingress(&ingress, "agent-report", idempotency_key) { - eprintln!("cmux-tui: journaling an agent report for {terminal_id} failed: {error}"); - } + self.append_journal_ingress(&ingress, "agent-report", idempotency_key) } /// Committed agent hook events double as the live agent-status feed: @@ -9375,6 +9373,7 @@ impl Mux { None, AgentReportOrigin::Direct, None, + None, ) } @@ -9401,6 +9400,39 @@ impl Mux { "source":source.as_str(), "source_session":session, }); + let (echo_sequence, echo_timestamp) = if source == AgentSource::Socket { + let terminal_id = { + let state_guard = self.state.lock().unwrap(); + let runtime = state_guard + .surfaces + .get(&surface) + .or_else(|| state_guard.terminal_runtime_by_id(surface)) + .with_context(|| format!("unknown surface {surface}"))?; + let identity = + runtime.resource_identity().context("surface has no resource identity")?; + let crate::ContentPublicId::Terminal(id) = identity.content_id else { + anyhow::bail!("surface {surface} is not a terminal") + }; + id + }; + let timestamp = now_ms(); + match self.agent_report_echo_key(&terminal_id, state, source, session.as_deref()) { + Some(key) => { + let commit = self.append_agent_report_echo( + &terminal_id, + state, + source, + session.as_deref(), + timestamp, + &key, + )?; + (Some(commit.sequence), Some(timestamp)) + } + None => (None, Some(timestamp)), + } + } else { + (journal_sequence, None) + }; let (_, record) = self.commit_agent_report( AgentReportTarget::Surface(surface), state, @@ -9411,9 +9443,10 @@ impl Mux { &fingerprint, sequence_lock_held, hook_state.as_ref(), - journal_sequence, + echo_sequence, origin, agent_adapter, + echo_timestamp, )?; let record = record.context("fresh raw agent report unexpectedly replayed")?; if source != AgentSource::Hook { @@ -9443,6 +9476,30 @@ impl Mux { "source":source.as_str(), "source_session":source_session, }); + let (journal_sequence, reported_at_ms) = if source == AgentSource::Socket { + let timestamp = now_ms(); + match self.agent_report_echo_key( + terminal_id, + agent_state, + source, + source_session.as_deref(), + ) { + Some(key) => { + let echo = self.append_agent_report_echo( + terminal_id, + agent_state, + source, + source_session.as_deref(), + timestamp, + &key, + )?; + (Some(echo.sequence), Some(timestamp)) + } + None => (None, Some(timestamp)), + } + } else { + (None, None) + }; let result = self.commit_agent_report( AgentReportTarget::Resource { selectors: &selectors, terminal_id }, agent_state, @@ -9453,9 +9510,10 @@ impl Mux { &fingerprint, false, None, - None, + journal_sequence, AgentReportOrigin::Direct, None, + reported_at_ms, ); if result.is_ok() && source != AgentSource::Hook { let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); @@ -9478,6 +9536,7 @@ impl Mux { journal_sequence: Option, origin: AgentReportOrigin, agent_adapter: Option, + reported_at_ms: Option, ) -> anyhow::Result<(ResourcePatchCommit, Option)> { // Hook state is accepted only through the validated journal ingress. // A direct Hook report has no durable event to replay, so allowing it @@ -9607,7 +9666,7 @@ impl Mux { let source_session = source_session.filter(|value| { !value.starts_with("cmux-hook-sequence:") && !value.starts_with("cmux-hook-ended:") }); - let now = now_ms(); + let now = reported_at_ms.unwrap_or_else(now_ms); let mut records = self.agent_records.lock().unwrap(); let socket_report_ignored = records.get(&terminal_id).is_some_and(|existing| { existing.source == AgentSource::Hook @@ -9725,24 +9784,6 @@ impl Mux { agent: agent.agent.as_deref().map(Arc::from), updated_at_ms: agent.updated_at_ms, }); - if origin == AgentReportOrigin::Direct - && agent.source == AgentSource::Socket - && let Some(idempotency_key) = self.agent_report_echo_key( - &agent.terminal_id, - agent.state, - agent.source, - agent.session.as_deref(), - ) - { - self.append_agent_report_echo( - &agent.terminal_id, - agent.state, - agent.source, - agent.session.as_deref(), - agent.updated_at_ms, - &idempotency_key, - ); - } } Ok((commit, Some(agent))) } From 656dffc9dcd7316db8196437bb9d17b5162d838d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:10:23 -0700 Subject: [PATCH 038/133] test(tui): reject direct hook compatibility reports --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 34 +++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 3481a66ba39e..0eedbf065079 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23182,15 +23182,17 @@ mod tests { .unwrap(); mux.append_journal_ingress(&end, "test", "authority-scope-end").unwrap(); - // Direct hook reports remain in the compatibility projection until - // their own terminal has a durable roster removal fence. - mux.report_agent( - compatibility_surface.id, - AgentState::Working, - AgentSource::Hook, - Some("compatibility-session".into()), - ) - .unwrap(); + // Direct hook reports are rejected because they have no durable + // journal event to replay after restart. + assert!( + mux.report_agent( + compatibility_surface.id, + AgentState::Working, + AgentSource::Hook, + Some("compatibility-session".into()), + ) + .is_err() + ); mux.shutdown(); drop(mux); @@ -23199,10 +23201,7 @@ mod tests { let compatibility_surface = reopened .resource_surface_for_terminal(&compatibility_terminal) .expect("compatibility terminal restored"); - let records = reopened.list_agents(Some(compatibility_surface), None); - assert_eq!(records.len(), 1); - assert_eq!(records[0].source, AgentSource::Hook); - assert_eq!(records[0].session.as_deref(), Some("compatibility-session")); + assert!(reopened.list_agents(Some(compatibility_surface), None).is_empty()); reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); @@ -24482,16 +24481,15 @@ mod tests { assert!(mux.surface(first.id).is_some()); assert!(mux.with_state(|state| state.surfaces.contains_key(&second.id))); - let detached_report = mux - .report_agent( + assert!( + mux.report_agent( first.id, AgentState::Blocked, AgentSource::Hook, Some("detached-hook".to_string()), ) - .expect("the terminal host identity remains valid without a view"); - assert_eq!(detached_report.state, AgentState::Blocked); - assert_eq!(detached_report.session.as_deref(), Some("detached-hook")); + .is_err() + ); mux.close_terminal(&host.terminal_id, &host.incarnation).unwrap(); assert!(mux.list_agents(None, None).is_empty()); From 5324764a3473aaab94d451a856d2d11a95713b45 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:20:48 -0700 Subject: [PATCH 039/133] fix(tui): restore ordered roster checkpoints --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 57 ++++++++---------------- 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 0eedbf065079..638df4f72489 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5680,21 +5680,18 @@ impl Mux { return; } loop { - let page = match self.workspace_registry.lock().unwrap().session_journal_after_subjects( - read_cursor, - 512, - &[crate::JournalSubject { - kind: "producer".into(), - id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), - }], - ) { + let page = match self + .workspace_registry + .lock() + .unwrap() + .session_journal_after(read_cursor, 512) + { Ok(page) => page, Err(error) => { eprintln!("cmux-tui: reading committed agent events back failed: {error}"); return; } }; - let scanned_through = page.scanned_through.min(commit.sequence); for record in page.records { if record.sequence <= read_cursor { continue; @@ -5712,40 +5709,24 @@ impl Mux { self.apply_roster_delta_to_record_cache(delta); } read_cursor = record.sequence; - // Checkpoint at a bounded journal interval. The in-memory - // reducer advances for every event, while durable snapshots - // lag by at most this interval and are rebuilt from the - // journal tail after restart. - if read_cursor % 64 == 0 { - let (cursor, snapshot) = { - let host = self.agent_roster.lock().unwrap(); - (host.cursor, host.roster.snapshot().to_string()) - }; - if let Err(error) = - self.workspace_registry.lock().unwrap().put_journal_reducer_state( - crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, - crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, - cursor, - &snapshot, - ) - { - eprintln!( - "cmux-tui: persisting the agent roster checkpoint failed: {error}" - ); - } + let (cursor, snapshot) = { + let host = self.agent_roster.lock().unwrap(); + (host.cursor, host.roster.snapshot().to_string()) + }; + if let Err(error) = + self.workspace_registry.lock().unwrap().put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ) + { + eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); } if read_cursor == commit.sequence { return; } } - // The filtered query scanned through this watermark. Advance - // over unrelated records, but never beyond the callback target. - if scanned_through > read_cursor { - read_cursor = scanned_through; - self.agent_roster.lock().unwrap().cursor = read_cursor; - } else { - return; - } } } From 2e1ce6a1a3dad58fb8465182c7b389c8f43f6a4f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:21:32 -0700 Subject: [PATCH 040/133] fix(tui): correct agent report compile references --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 638df4f72489..e81e31c05d55 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9354,7 +9354,6 @@ impl Mux { None, AgentReportOrigin::Direct, None, - None, ) } @@ -9391,7 +9390,7 @@ impl Mux { .with_context(|| format!("unknown surface {surface}"))?; let identity = runtime.resource_identity().context("surface has no resource identity")?; - let crate::ContentPublicId::Terminal(id) = identity.content_id else { + let crate::resource::ContentPublicId::Terminal(id) = identity.content_id else { anyhow::bail!("surface {surface} is not a terminal") }; id From 703a40c26b277b6c4b13ac433cfe9557c98525c0 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:13:04 -0700 Subject: [PATCH 041/133] fix(tui): commit agent echoes with projections --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 69 ++++++++++++++----- .../workspace_registry/journal_extensions.rs | 2 +- .../src/workspace_registry/resource_store.rs | 51 ++++++++++++++ 3 files changed, 104 insertions(+), 18 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e81e31c05d55..50e9aa7df714 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5811,15 +5811,15 @@ impl Mux { /// Echo direct reports into the journal so a restart can rebuild the /// socket-owned roster entry from the same durable event stream. - fn append_agent_report_echo( + fn agent_report_echo_ingress( &self, terminal_id: &TerminalPublicId, state: AgentState, source: AgentSource, session: Option<&str>, updated_at_ms: u64, - idempotency_key: &str, - ) -> anyhow::Result { + _idempotency_key: &str, + ) -> anyhow::Result { use crate::journal_reducers::{SOCKET_REPORT_ADAPTER, SOCKET_REPORT_NATIVE_EVENT}; let ingress = crate::JournalIngress { producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), @@ -5847,7 +5847,7 @@ impl Mux { causation_id: None, correlation_id: None, }; - self.append_journal_ingress(&ingress, "agent-report", idempotency_key) + Ok(ingress) } /// Committed agent hook events double as the live agent-status feed: @@ -9380,7 +9380,7 @@ impl Mux { "source":source.as_str(), "source_session":session, }); - let (echo_sequence, echo_timestamp) = if source == AgentSource::Socket { + let (echo_sequence, echo_timestamp, echo_ingress) = if source == AgentSource::Socket { let terminal_id = { let state_guard = self.state.lock().unwrap(); let runtime = state_guard @@ -9398,7 +9398,7 @@ impl Mux { let timestamp = now_ms(); match self.agent_report_echo_key(&terminal_id, state, source, session.as_deref()) { Some(key) => { - let commit = self.append_agent_report_echo( + let ingress = self.agent_report_echo_ingress( &terminal_id, state, source, @@ -9406,12 +9406,12 @@ impl Mux { timestamp, &key, )?; - (Some(commit.sequence), Some(timestamp)) + (None, Some(timestamp), Some((ingress, key))) } - None => (None, Some(timestamp)), + None => (None, Some(timestamp), None), } } else { - (journal_sequence, None) + (journal_sequence, None, None) }; let (_, record) = self.commit_agent_report( AgentReportTarget::Surface(surface), @@ -9427,6 +9427,7 @@ impl Mux { origin, agent_adapter, echo_timestamp, + echo_ingress.as_ref(), )?; let record = record.context("fresh raw agent report unexpectedly replayed")?; if source != AgentSource::Hook { @@ -9456,7 +9457,7 @@ impl Mux { "source":source.as_str(), "source_session":source_session, }); - let (journal_sequence, reported_at_ms) = if source == AgentSource::Socket { + let (journal_sequence, reported_at_ms, echo_ingress) = if source == AgentSource::Socket { let timestamp = now_ms(); match self.agent_report_echo_key( terminal_id, @@ -9465,7 +9466,7 @@ impl Mux { source_session.as_deref(), ) { Some(key) => { - let echo = self.append_agent_report_echo( + let ingress = self.agent_report_echo_ingress( terminal_id, agent_state, source, @@ -9473,12 +9474,12 @@ impl Mux { timestamp, &key, )?; - (Some(echo.sequence), Some(timestamp)) + (None, Some(timestamp), Some((ingress, key))) } - None => (None, Some(timestamp)), + None => (None, Some(timestamp), None), } } else { - (None, None) + (None, None, None) }; let result = self.commit_agent_report( AgentReportTarget::Resource { selectors: &selectors, terminal_id }, @@ -9494,6 +9495,7 @@ impl Mux { AgentReportOrigin::Direct, None, reported_at_ms, + echo_ingress.as_ref(), ); if result.is_ok() && source != AgentSource::Hook { let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); @@ -9517,6 +9519,7 @@ impl Mux { origin: AgentReportOrigin, agent_adapter: Option, reported_at_ms: Option, + echo_ingress: Option<&(crate::JournalIngress, String)>, ) -> anyhow::Result<(ResourcePatchCommit, Option)> { // Hook state is accepted only through the validated journal ingress. // A direct Hook report has no durable event to replay, so allowing it @@ -9703,8 +9706,24 @@ impl Mux { "value":public_value, }]) }; - let commit = match journal_sequence { - Some(sequence) => registry.commit_agent_projection_with_hook_state_and_sequence( + let commit = match (journal_sequence, echo_ingress) { + (None, Some((ingress, key))) => { + let validated = self.journal_kernel.validate_ingress(ingress)?; + registry.commit_agent_projection_with_hook_state_and_journal( + mutation, + fingerprint, + expected_revision, + &terminal_id, + &value, + &deltas, + effective_hook_state, + ingress, + &validated, + "agent-report", + key, + )? + } + (Some(sequence), _) => registry.commit_agent_projection_with_hook_state_and_sequence( mutation, fingerprint, expected_revision, @@ -9714,7 +9733,7 @@ impl Mux { effective_hook_state, sequence, )?, - None => registry.commit_agent_projection_with_hook_state( + (None, None) => registry.commit_agent_projection_with_hook_state( mutation, fingerprint, expected_revision, @@ -24386,6 +24405,13 @@ mod tests { let surface = mux.new_workspace(None, None).unwrap(); let revision = mux.with_state(|state| state.resource_revision); let epoch = mux.resource_event_epoch(); + let journal_head = mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + .unwrap() + .head_sequence; mux.workspace_registry.lock().unwrap().set_resource_patch_failure(true).unwrap(); let error = mux @@ -24401,6 +24427,15 @@ mod tests { assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 0); assert_eq!(mux.with_state(|state| state.resource_revision), revision); assert_eq!(mux.resource_event_epoch(), epoch); + assert_eq!( + mux.workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + .unwrap() + .head_sequence, + journal_head + ); assert!(mux.resource_events_after(revision).unwrap().batches.is_empty()); mux.workspace_registry.lock().unwrap().set_resource_patch_failure(false).unwrap(); } diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs index c70c1082c0d0..8e6eefe5e014 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs @@ -1407,7 +1407,7 @@ impl WorkspaceRegistry { } } -fn append_journal_ingress_transaction( +pub(crate) fn append_journal_ingress_transaction( tx: &Transaction<'_>, ingress: &JournalIngress, validated: &crate::journal_kernel::ValidatedJournalIngress, diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 7340c8088333..32f90bc56f2d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -842,6 +842,7 @@ impl WorkspaceRegistry { deltas, hook_state, None, + None, ) } @@ -865,6 +866,34 @@ impl WorkspaceRegistry { deltas, hook_state, Some(journal_sequence), + None, + ) + } + + pub fn commit_agent_projection_with_hook_state_and_journal( + &mut self, + mutation: &WorkspaceMutation, + fingerprint: &Value, + expected_revision: Option, + terminal_id: &TerminalPublicId, + result: &Value, + deltas: &Value, + hook_state: Option<&AgentHookProjectionState>, + ingress: &crate::JournalIngress, + validated: &crate::journal_kernel::ValidatedJournalIngress, + origin: &str, + idempotency_key: &str, + ) -> anyhow::Result { + self.commit_agent_projection_inner( + mutation, + fingerprint, + expected_revision, + terminal_id, + result, + deltas, + hook_state, + None, + Some((ingress, validated, origin, idempotency_key)), ) } @@ -899,6 +928,7 @@ impl WorkspaceRegistry { deltas, None, None, + None, ) } @@ -913,6 +943,12 @@ impl WorkspaceRegistry { deltas: &Value, hook_state: Option<&AgentHookProjectionState>, journal_sequence: Option, + journal_echo: Option<( + &crate::JournalIngress, + &crate::journal_kernel::ValidatedJournalIngress, + &str, + &str, + )>, ) -> anyhow::Result { const OPERATION: &str = "agent.report"; validate_identifier("mutation id", &mutation.id)?; @@ -924,6 +960,21 @@ impl WorkspaceRegistry { let fingerprint = canonical_json(fingerprint)?; let result_json = canonical_json(result)?; let tx = self.connection.transaction()?; + let journal_sequence = + if let Some((ingress, validated, origin, idempotency_key)) = journal_echo { + Some( + super::journal_extensions::append_journal_ingress_transaction( + &tx, + ingress, + validated, + origin, + idempotency_key, + )? + .sequence, + ) + } else { + journal_sequence + }; if let Some(replayed) = resource_patch_replay(&tx, mutation, OPERATION, &fingerprint)? { if let Some(sequence) = journal_sequence { advance_agent_hook_apply_cursor_transaction(&tx, sequence)?; From ce89026f37d8110bff0ca889b73fbfd9409ffc67 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:19:12 -0700 Subject: [PATCH 042/133] fix(tui): borrow terminal identity during report --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 50e9aa7df714..f607a76599f2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9390,10 +9390,10 @@ impl Mux { .with_context(|| format!("unknown surface {surface}"))?; let identity = runtime.resource_identity().context("surface has no resource identity")?; - let crate::resource::ContentPublicId::Terminal(id) = identity.content_id else { + let crate::resource::ContentPublicId::Terminal(id) = &identity.content_id else { anyhow::bail!("surface {surface} is not a terminal") }; - id + id.clone() }; let timestamp = now_ms(); match self.agent_report_echo_key(&terminal_id, state, source, session.as_deref()) { From e2b6621471e0cfd001ab6835954c1745101040a1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:21:57 -0700 Subject: [PATCH 043/133] fix(tui): fold atomic agent echoes into roster --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f607a76599f2..d1ce6395084d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9761,9 +9761,18 @@ impl Mux { if !commit.replayed { records.insert(terminal_id.clone(), record.clone()); } + let echo_fold_sequence = echo_ingress + .filter(|_| !commit.replayed) + .and_then(|_| registry.agent_hook_apply_cursor().ok()); drop(records); drop(state); drop(registry); + if let (Some((ingress, _)), Some(sequence)) = (echo_ingress, echo_fold_sequence) { + self.fold_agent_roster( + ingress, + &crate::JournalAppendCommit { sequence, event_id: String::new(), replayed: false }, + ); + } let agent = AgentRecord { surface, terminal_id, From a6c2948d10dd5d90c0362225ea4eac0b66651f90 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:22:27 -0700 Subject: [PATCH 044/133] test(tui): verify socket report roster coalescing --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d1ce6395084d..68cbea87d8ec 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24449,6 +24449,56 @@ mod tests { mux.workspace_registry.lock().unwrap().set_resource_patch_failure(false).unwrap(); } + #[test] + fn socket_agent_report_folds_roster_and_coalesces_repeated_state() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let before_cursor = mux.agent_roster.lock().unwrap().cursor; + let before_head = mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + .unwrap() + .head_sequence; + + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("repeat-session".into()), + ) + .unwrap(); + let after_first_cursor = mux.agent_roster.lock().unwrap().cursor; + let after_first_head = mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + .unwrap() + .head_sequence; + assert!(after_first_cursor > before_cursor); + assert!(after_first_head > before_head); + + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("repeat-session".into()), + ) + .unwrap(); + assert_eq!(mux.agent_roster.lock().unwrap().cursor, after_first_cursor); + assert_eq!( + mux.workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + .unwrap() + .head_sequence, + after_first_head + ); + } + #[test] fn agent_reports_require_a_terminal_and_never_create_a_default_projection() { let mux = test_mux(); From bbefc529bb0824c08c4eef4ae60de4a8d09e1d7a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:34:27 -0700 Subject: [PATCH 045/133] fix(tui): bound roster recovery folds --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 68cbea87d8ec..1ae2f6a886b8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1126,10 +1126,14 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result= commit.sequence { + return; } } } From 27267fe8f62e693fac944bdfbf92a918e9b36cfa Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:34:55 -0700 Subject: [PATCH 046/133] test(tui): route compatibility hook through journal --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 1ae2f6a886b8..599fca4f59a4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23140,13 +23140,14 @@ mod tests { let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); let surface = mux.new_workspace(None, None).unwrap(); let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); - mux.report_agent( - surface.id, - AgentState::Working, - AgentSource::Hook, - Some("compatibility-session".into()), + let started = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"compatibility-session"}), ) .unwrap(); + mux.append_journal_ingress(&started, "test", "unsupported-hook-start").unwrap(); let child = crate::agent_hooks::agent_hook_journal_ingress( "claude", From 3e70a0941b51c788f1269affdc5de0df8594c09b Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:44:58 -0700 Subject: [PATCH 047/133] fix(tui): bound startup roster replay --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 599fca4f59a4..44722d430304 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1097,6 +1097,7 @@ struct AgentRosterHost { } fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { + const MAX_REPLAY_RECORDS: usize = 100_000; use crate::journal_reducers::{ AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, AgentRoster, RosterEvent, }; @@ -1121,12 +1122,18 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result AgentRosterHost::default(), }; let initial_cursor = host.cursor; + let mut replayed_records = 0usize; loop { let page = registry.session_journal_after(host.cursor, 512)?; if page.records.is_empty() { break; } let previous_cursor = host.cursor; + replayed_records = replayed_records.saturating_add(page.records.len()); + anyhow::ensure!( + replayed_records <= MAX_REPLAY_RECORDS, + "agent roster replay exceeds {MAX_REPLAY_RECORDS} records; reducer checkpoint required" + ); for record in &page.records { host.roster.apply(&RosterEvent::from_record(record)); host.cursor = host.cursor.max(record.sequence); From a47269cc48fc3a7cdd3c5517fd54645b1b07b7d7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:06:05 -0700 Subject: [PATCH 048/133] test(tui): cover roster purge snapshot failure --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 44722d430304..acc05b18a8fa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24378,6 +24378,66 @@ mod tests { mux.shutdown(); } + #[test] + fn terminal_side_tables_clear_and_roster_retirement_retries_after_snapshot_failure() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("snapshot-failure".into()), + ) + .unwrap(); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({}), + ) + .unwrap(); + mux.workspace_registry + .lock() + .unwrap() + .enqueue_agent_hook_pending( + &ingress.producer_id, + "test", + "pending-before-snapshot-failure", + 1, + &ingress, + AGENT_HOOK_RETRY_ERROR, + crate::workspace_registry::AgentHookRetryClass::Transient, + ) + .unwrap(); + + mux.workspace_registry.lock().unwrap().set_journal_reducer_state_failure(true).unwrap(); + let failed = mux.purge_terminal_side_tables(&terminal_id); + assert!(failed.is_err(), "the durable snapshot error must remain observable"); + assert!( + !mux.agent_records.lock().unwrap().contains_key(&terminal_id), + "a failed roster snapshot must not retain the compatibility record" + ); + assert!( + mux.workspace_registry + .lock() + .unwrap() + .pending_agent_hook_projections() + .unwrap() + .is_empty(), + "durable pending hook receipts must still be purged" + ); + assert!( + !mux.agent_roster.lock().unwrap().roster.is_retired(terminal_id.as_str()), + "the failed tombstone must remain retryable" + ); + + mux.workspace_registry.lock().unwrap().set_journal_reducer_state_failure(false).unwrap(); + mux.purge_terminal_side_tables(&terminal_id).unwrap(); + assert!(mux.agent_roster.lock().unwrap().roster.is_retired(terminal_id.as_str())); + mux.shutdown(); + } + #[test] fn startup_reconciliation_removes_stale_ended_agent_projection() { let root = std::env::temp_dir().join(format!( From ff62bffefd389676c27f8a7b50fe8e8ce7b93663 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:06:19 -0700 Subject: [PATCH 049/133] fix(tui): clear side tables after roster snapshot failure --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index acc05b18a8fa..8641b30f7d78 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9862,7 +9862,7 @@ impl Mux { let retired = host.roster.retire_terminal(terminal_id.as_str(), journal_cursor); retired.then(|| (previous, host.cursor, host.roster.snapshot().to_string())) }; - if let Some((previous, cursor, snapshot)) = retired { + let snapshot_error = if let Some((previous, cursor, snapshot)) = retired { let persisted = self.workspace_registry.lock().unwrap().put_journal_reducer_state( crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, @@ -9876,13 +9876,17 @@ impl Mux { self.report_internal_diagnostic( "terminal agent retirement deferred before snapshot", ); - return Err(error).context("persist agent roster tombstone"); + Some(error.context("persist agent roster tombstone")) + } else { + None } - } + } else { + None + }; self.agent_hook_fences.lock().unwrap().remove(terminal_id); self.agent_records.lock().unwrap().remove(terminal_id); self.terminal_notifications.lock().unwrap().remove(terminal_id); - Ok(()) + snapshot_error.map_or(Ok(()), Err) } fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) -> anyhow::Result<()> { From 9ca0544b00a833474ed0f58b8bb3282df2051f94 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:05:55 -0700 Subject: [PATCH 050/133] test(tui): cover hook-owned socket echo coalescing --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8641b30f7d78..1bcc3b4ac556 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23143,6 +23143,62 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn socket_reports_do_not_echo_when_hook_owns_roster_entry() { + let root = std::env::temp_dir() + .join(format!( + "cmux-roster-hook-echo-{}", + crate::workspace_registry::new_uuid_v4() + )); + let mux = Mux::open_persistent("roster-hook-echo", SurfaceOptions::default(), &root) + .unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Hook, + Some("hook".into()), + ) + .unwrap(); + let socket_echoes = || { + mux.session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count() + }; + + assert_eq!(socket_echoes(), 0); + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("poll".into()), + ) + .unwrap(); + mux.report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("poll".into()), + ) + .unwrap(); + assert_eq!(socket_echoes(), 0, "hook-owned reports must not append socket echoes"); + + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn unsupported_hook_events_do_not_make_compatibility_projection_authoritative() { let root = std::env::temp_dir() From 8e663624626a362cd2ff1877f399b061bea0be50 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:07:54 -0700 Subject: [PATCH 051/133] fix(tui): skip socket echoes for hook-owned roster entries --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 42 ++++++++---------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 1bcc3b4ac556..7a37ce3ec797 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5805,7 +5805,11 @@ impl Mux { && previous_session.as_deref() == session }, ); - if unchanged { + let hook_owned_socket_report = + previous.as_ref().is_some_and(|(_, previous_source, _, _)| { + previous_source == AgentSource::Hook.as_str() && source == AgentSource::Socket + }); + if unchanged || hook_owned_socket_report { return None; } let previous = previous @@ -23146,21 +23150,13 @@ mod tests { #[test] fn socket_reports_do_not_echo_when_hook_owns_roster_entry() { let root = std::env::temp_dir() - .join(format!( - "cmux-roster-hook-echo-{}", - crate::workspace_registry::new_uuid_v4() - )); - let mux = Mux::open_persistent("roster-hook-echo", SurfaceOptions::default(), &root) - .unwrap(); + .join(format!("cmux-roster-hook-echo-{}", crate::workspace_registry::new_uuid_v4())); + let mux = + Mux::open_persistent("roster-hook-echo", SurfaceOptions::default(), &root).unwrap(); let surface = mux.new_workspace(None, None).unwrap(); - mux.report_agent( - surface.id, - AgentState::Working, - AgentSource::Hook, - Some("hook".into()), - ) - .unwrap(); + mux.report_agent(surface.id, AgentState::Working, AgentSource::Hook, Some("hook".into())) + .unwrap(); let socket_echoes = || { mux.session_journal_after(0, 1024) .unwrap() @@ -23178,20 +23174,10 @@ mod tests { }; assert_eq!(socket_echoes(), 0); - mux.report_agent( - surface.id, - AgentState::Working, - AgentSource::Socket, - Some("poll".into()), - ) - .unwrap(); - mux.report_agent( - surface.id, - AgentState::Working, - AgentSource::Socket, - Some("poll".into()), - ) - .unwrap(); + mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) + .unwrap(); + mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) + .unwrap(); assert_eq!(socket_echoes(), 0, "hook-owned reports must not append socket echoes"); mux.shutdown(); From 506b90c2fb8ad9649412975c5730e64173527483 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:06:52 -0700 Subject: [PATCH 052/133] fix(tui): remove arbitrary roster replay cap --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 50 ++++++++++++++++++++---- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7a37ce3ec797..38fbb84f21ea 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1097,7 +1097,6 @@ struct AgentRosterHost { } fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { - const MAX_REPLAY_RECORDS: usize = 100_000; use crate::journal_reducers::{ AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, AgentRoster, RosterEvent, }; @@ -1122,18 +1121,12 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result AgentRosterHost::default(), }; let initial_cursor = host.cursor; - let mut replayed_records = 0usize; loop { let page = registry.session_journal_after(host.cursor, 512)?; if page.records.is_empty() { break; } let previous_cursor = host.cursor; - replayed_records = replayed_records.saturating_add(page.records.len()); - anyhow::ensure!( - replayed_records <= MAX_REPLAY_RECORDS, - "agent roster replay exceeds {MAX_REPLAY_RECORDS} records; reducer checkpoint required" - ); for record in &page.records { host.roster.apply(&RosterEvent::from_record(record)); host.cursor = host.cursor.max(record.sequence); @@ -23017,6 +23010,49 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn roster_restore_replays_beyond_legacy_record_cap() { + let root = std::env::temp_dir().join(format!( + "cmux-roster-large-replay-{}", + crate::workspace_registry::new_uuid_v4() + )); + let mux = Mux::open_persistent("roster-large-replay", SurfaceOptions::default(), &root) + .unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("terminal"); + let event = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"large-replay"}), + ) + .unwrap(); + let validated = mux.journal_kernel.validate_ingress(&event).unwrap(); + for index in 0..100_001 { + mux.workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&event, &validated, "test", &format!("large-replay-{index}")) + .unwrap(); + } + mux.shutdown(); + drop(mux); + + let registry = WorkspaceRegistry::open(&root, "roster-large-replay").unwrap(); + registry + .put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + 0, + "{malformed", + ) + .unwrap(); + let restored = restore_agent_roster(®istry).unwrap(); + assert_eq!(restored.cursor, registry.session_journal_after(0, 1).unwrap().head_sequence); + assert!(restored.roster.entries.contains_key(terminal_id.as_str())); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn retired_terminal_cannot_be_resurrected_by_a_delayed_journal_event() { let root = std::env::temp_dir() From 9ab8fe24e8b8269efa72c0b6859423c54aaa433e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:23:58 -0700 Subject: [PATCH 053/133] test: guard roster cache timestamp ordering --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 38fbb84f21ea..36441d3d9f75 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22896,6 +22896,26 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn delayed_roster_fold_cannot_overwrite_newer_direct_timestamp() { + use crate::journal_reducers::{RosterDelta, RosterEntry}; + let root = std::env::temp_dir().join(format!("cmux-roster-monotonic-{}", crate::workspace_registry::new_uuid_v4())); + let mux = Mux::open_persistent("roster-monotonic", SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().unwrap(); + mux.agent_records.lock().unwrap().insert(terminal_id.clone(), TerminalAgentRecord { + state: AgentState::Working, source: AgentSource::Socket, session: None, agent: None, updated_at_ms: 200, + }); + mux.apply_roster_delta_to_record_cache(RosterDelta::Upsert { + terminal_id: terminal_id.to_string(), + entry: RosterEntry { state: "idle".into(), source: "socket".into(), session: None, agent: None, updated_at_ms: 100 }, + }); + assert_eq!(mux.agent_records.lock().unwrap().get(&terminal_id).unwrap().updated_at_ms, 200); + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn roster_fold_pages_to_a_target_beyond_the_first_journal_page() { let root = std::env::temp_dir().join(format!( From 79389102125f906115bba2f157cd6589506b3a24 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:24:06 -0700 Subject: [PATCH 054/133] fix: preserve newer agent report timestamps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 36441d3d9f75..5c604aaff139 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5749,6 +5749,12 @@ impl Mux { match delta { RosterDelta::Upsert { terminal_id, entry } => { let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) else { return }; + if records + .get(&terminal_id) + .is_some_and(|existing| existing.updated_at_ms > entry.updated_at_ms) + { + return; + } records.insert( terminal_id, TerminalAgentRecord { From 71fbf1c8e74ab252e5d8cdc0ffea8901fb4b7afd Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:30:09 -0700 Subject: [PATCH 055/133] test: retain all retired terminal fences --- cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 31c405bc2a4d..d5207a213242 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -682,11 +682,11 @@ mod tests { } #[test] - fn terminal_fences_are_bounded() { - const EXPECTED_FENCE_LIMIT: usize = 1_024; + fn terminal_fences_are_retained_until_journal_compaction() { + const RETIRED_TERMINAL_COUNT: usize = 2_048; let payload = json!({}); let mut live = AgentRoster::default(); - for sequence in 1..=EXPECTED_FENCE_LIMIT * 2 { + for sequence in 1..=RETIRED_TERMINAL_COUNT * 2 { let subjects = terminal_subject("long_lived"); live.apply(&hook_event( sequence as u64, @@ -698,7 +698,7 @@ mod tests { assert_eq!(live.hook_fences.len(), 1); let mut retired = AgentRoster::default(); - for index in 0..(EXPECTED_FENCE_LIMIT * 2) { + for index in 0..RETIRED_TERMINAL_COUNT { let terminal_id = format!("retired_{index}"); let subjects = terminal_subject(&terminal_id); let sequence = index as u64 * 2 + 1; From 88d5c8c7f7aea30d6aba6c0a3e6a9f9468d3f6bb Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:30:22 -0700 Subject: [PATCH 056/133] fix: keep retirement fences until journal compaction --- .../cmux-tui-core/src/journal_reducers.rs | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index d5207a213242..fa01b3d92824 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -34,9 +34,10 @@ pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 5; /// Retirement tombstones protect delayed journal rows after a terminal leaves -/// the resource tree. Keep a bounded safety net. Durable startup reconciliation -/// removes entries for terminals that are confirmed gone. -pub(crate) const MAX_RETIRED_TERMINAL_FENCES: usize = 1_024; +/// the resource tree. Keep each exact tombstone while its journal rows may be +/// retained. A future journal-compaction watermark can safely remove older +/// tombstones; count-based eviction would allow delayed rows to resurrect a +/// retired terminal. /// The adapter id and native event the socket report path uses for its echo /// journal events. The echo carries the explicit state in `normalized`, so @@ -338,23 +339,7 @@ impl AgentRoster { true } }; - let pruned = self.prune_retired_terminals(); - removed_entry || removed_fence || retired_changed || pruned - } - - fn prune_retired_terminals(&mut self) -> bool { - let mut changed = false; - while self.retired_terminals.len() > MAX_RETIRED_TERMINAL_FENCES { - let oldest = self - .retired_terminals - .iter() - .min_by_key(|(terminal_id, sequence)| (**sequence, terminal_id.as_str())) - .map(|(terminal_id, _)| terminal_id.clone()); - let Some(oldest) = oldest else { break }; - self.retired_terminals.remove(&oldest); - changed = true; - } - changed + removed_entry || removed_fence || retired_changed } pub(crate) fn is_retired(&self, terminal_id: &str) -> bool { @@ -707,7 +692,11 @@ mod tests { retired.retire_terminal(&terminal_id, sequence); } assert!(retired.hook_fences.is_empty()); - assert!(retired.retired_terminals.len() <= EXPECTED_FENCE_LIMIT); + // A delayed journal row can arrive at any time while its sequence is + // retained. Keep every exact tombstone until a journal-compaction + // watermark exists; count-based eviction permits resurrection. + assert_eq!(retired.retired_terminals.len(), RETIRED_TERMINAL_COUNT); + assert_eq!(retired.retired_terminals.get("retired_0"), Some(&1)); assert_eq!(retired.retired_terminals.get("retired_2047"), Some(&4_095)); } } From 46e1e4a228abdcec8e13d7588b49d197db71afa4 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:30:50 -0700 Subject: [PATCH 057/133] style: format roster regression tests --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 44 +++++++++++++++++------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5c604aaff139..f94afd71c76c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22905,16 +22905,31 @@ mod tests { #[test] fn delayed_roster_fold_cannot_overwrite_newer_direct_timestamp() { use crate::journal_reducers::{RosterDelta, RosterEntry}; - let root = std::env::temp_dir().join(format!("cmux-roster-monotonic-{}", crate::workspace_registry::new_uuid_v4())); - let mux = Mux::open_persistent("roster-monotonic", SurfaceOptions::default(), &root).unwrap(); + let root = std::env::temp_dir() + .join(format!("cmux-roster-monotonic-{}", crate::workspace_registry::new_uuid_v4())); + let mux = + Mux::open_persistent("roster-monotonic", SurfaceOptions::default(), &root).unwrap(); let surface = mux.new_workspace(None, None).unwrap(); let terminal_id = surface.terminal_public_id().cloned().unwrap(); - mux.agent_records.lock().unwrap().insert(terminal_id.clone(), TerminalAgentRecord { - state: AgentState::Working, source: AgentSource::Socket, session: None, agent: None, updated_at_ms: 200, - }); + mux.agent_records.lock().unwrap().insert( + terminal_id.clone(), + TerminalAgentRecord { + state: AgentState::Working, + source: AgentSource::Socket, + session: None, + agent: None, + updated_at_ms: 200, + }, + ); mux.apply_roster_delta_to_record_cache(RosterDelta::Upsert { terminal_id: terminal_id.to_string(), - entry: RosterEntry { state: "idle".into(), source: "socket".into(), session: None, agent: None, updated_at_ms: 100 }, + entry: RosterEntry { + state: "idle".into(), + source: "socket".into(), + session: None, + agent: None, + updated_at_ms: 100, + }, }); assert_eq!(mux.agent_records.lock().unwrap().get(&terminal_id).unwrap().updated_at_ms, 200); mux.shutdown(); @@ -23038,12 +23053,10 @@ mod tests { #[test] fn roster_restore_replays_beyond_legacy_record_cap() { - let root = std::env::temp_dir().join(format!( - "cmux-roster-large-replay-{}", - crate::workspace_registry::new_uuid_v4() - )); - let mux = Mux::open_persistent("roster-large-replay", SurfaceOptions::default(), &root) - .unwrap(); + let root = std::env::temp_dir() + .join(format!("cmux-roster-large-replay-{}", crate::workspace_registry::new_uuid_v4())); + let mux = + Mux::open_persistent("roster-large-replay", SurfaceOptions::default(), &root).unwrap(); let surface = mux.new_workspace(None, None).unwrap(); let terminal_id = surface.terminal_public_id().cloned().expect("terminal"); let event = crate::agent_hooks::agent_hook_journal_ingress( @@ -23058,7 +23071,12 @@ mod tests { mux.workspace_registry .lock() .unwrap() - .append_journal_ingress(&event, &validated, "test", &format!("large-replay-{index}")) + .append_journal_ingress( + &event, + &validated, + "test", + &format!("large-replay-{index}"), + ) .unwrap(); } mux.shutdown(); From 8fbbc61838c2a72bedf9acfe493b3b09ac38215b Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:50:06 -0700 Subject: [PATCH 058/133] test(tui): keep hook cursor behind unapplied socket echo --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f94afd71c76c..5e5c317929c7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24271,6 +24271,42 @@ mod tests { assert_eq!(mux.list_agents(Some(surface.id), None).len(), 1); } + #[test] + fn socket_echo_does_not_advance_cursor_past_unapplied_hook() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({}), + ) + .unwrap(); + let validated = mux.journal_kernel.validate_ingress(&ingress).unwrap(); + let hook_commit = mux + .workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&ingress, &validated, "unapplied-hook", "hook-1") + .unwrap(); + assert_eq!(hook_commit.sequence, 1); + assert_eq!( + mux.workspace_registry.lock().unwrap().agent_hook_apply_cursor().unwrap(), + 0 + ); + + mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) + .unwrap(); + + assert_eq!( + mux.workspace_registry.lock().unwrap().agent_hook_apply_cursor().unwrap(), + 0, + "a socket echo must not consume an unapplied hook sequence" + ); + assert_eq!(mux.session_journal_after(0, 16).unwrap().head_sequence, 2); + } + #[test] fn reopened_ended_hook_fence_rejects_late_events() { let root = std::env::temp_dir() From 5b12a6d7230dbc51a5e770772b81a491d8787dc9 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:58:21 -0700 Subject: [PATCH 059/133] fix(tui): keep socket echoes out of hook cursor --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 56 ++++++++++--------- .../src/workspace_registry/resource_store.rs | 27 ++++++--- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5e5c317929c7..382a5858b89d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9729,7 +9729,7 @@ impl Mux { "value":public_value, }]) }; - let commit = match (journal_sequence, echo_ingress) { + let (commit, echo_sequence) = match (journal_sequence, echo_ingress) { (None, Some((ingress, key))) => { let validated = self.journal_kernel.validate_ingress(ingress)?; registry.commit_agent_projection_with_hook_state_and_journal( @@ -9746,25 +9746,31 @@ impl Mux { key, )? } - (Some(sequence), _) => registry.commit_agent_projection_with_hook_state_and_sequence( - mutation, - fingerprint, - expected_revision, - &terminal_id, - &value, - &deltas, - effective_hook_state, - sequence, - )?, - (None, None) => registry.commit_agent_projection_with_hook_state( - mutation, - fingerprint, - expected_revision, - &terminal_id, - &value, - &deltas, - effective_hook_state, - )?, + (Some(sequence), _) => ( + registry.commit_agent_projection_with_hook_state_and_sequence( + mutation, + fingerprint, + expected_revision, + &terminal_id, + &value, + &deltas, + effective_hook_state, + sequence, + )?, + None, + ), + (None, None) => ( + registry.commit_agent_projection_with_hook_state( + mutation, + fingerprint, + expected_revision, + &terminal_id, + &value, + &deltas, + effective_hook_state, + )?, + None, + ), }; if !commit.replayed { if let (Some(direct_state), Some(sequence_guard)) = @@ -9784,9 +9790,8 @@ impl Mux { if !commit.replayed { records.insert(terminal_id.clone(), record.clone()); } - let echo_fold_sequence = echo_ingress - .filter(|_| !commit.replayed) - .and_then(|_| registry.agent_hook_apply_cursor().ok()); + let echo_fold_sequence = + echo_ingress.filter(|_| !commit.replayed).and_then(|_| echo_sequence); drop(records); drop(state); drop(registry); @@ -24291,10 +24296,7 @@ mod tests { .append_journal_ingress(&ingress, &validated, "unapplied-hook", "hook-1") .unwrap(); assert_eq!(hook_commit.sequence, 1); - assert_eq!( - mux.workspace_registry.lock().unwrap().agent_hook_apply_cursor().unwrap(), - 0 - ); + assert_eq!(mux.workspace_registry.lock().unwrap().agent_hook_apply_cursor().unwrap(), 0); mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, Some("poll".into())) .unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 32f90bc56f2d..cde19d206dd8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -844,6 +844,7 @@ impl WorkspaceRegistry { None, None, ) + .map(|(commit, _)| commit) } pub fn commit_agent_projection_with_hook_state_and_sequence( @@ -868,6 +869,7 @@ impl WorkspaceRegistry { Some(journal_sequence), None, ) + .map(|(commit, _)| commit) } pub fn commit_agent_projection_with_hook_state_and_journal( @@ -883,8 +885,8 @@ impl WorkspaceRegistry { validated: &crate::journal_kernel::ValidatedJournalIngress, origin: &str, idempotency_key: &str, - ) -> anyhow::Result { - self.commit_agent_projection_inner( + ) -> anyhow::Result<(ResourcePatchCommit, u64)> { + let (commit, echo_sequence) = self.commit_agent_projection_inner( mutation, fingerprint, expected_revision, @@ -894,7 +896,10 @@ impl WorkspaceRegistry { hook_state, None, Some((ingress, validated, origin, idempotency_key)), - ) + )?; + let echo_sequence = + echo_sequence.context("agent journal echo did not produce a committed sequence")?; + Ok((commit, echo_sequence)) } pub fn replay_resource_patch( @@ -930,6 +935,7 @@ impl WorkspaceRegistry { None, None, ) + .map(|(commit, _)| commit) } #[allow(clippy::too_many_arguments)] @@ -949,7 +955,7 @@ impl WorkspaceRegistry { &str, &str, )>, - ) -> anyhow::Result { + ) -> anyhow::Result<(ResourcePatchCommit, Option)> { const OPERATION: &str = "agent.report"; validate_identifier("mutation id", &mutation.id)?; validate_identifier("mutation origin", &mutation.origin)?; @@ -960,7 +966,7 @@ impl WorkspaceRegistry { let fingerprint = canonical_json(fingerprint)?; let result_json = canonical_json(result)?; let tx = self.connection.transaction()?; - let journal_sequence = + let echo_sequence = if let Some((ingress, validated, origin, idempotency_key)) = journal_echo { Some( super::journal_extensions::append_journal_ingress_transaction( @@ -973,14 +979,16 @@ impl WorkspaceRegistry { .sequence, ) } else { - journal_sequence + None }; if let Some(replayed) = resource_patch_replay(&tx, mutation, OPERATION, &fingerprint)? { if let Some(sequence) = journal_sequence { advance_agent_hook_apply_cursor_transaction(&tx, sequence)?; + } + if journal_sequence.is_some() || echo_sequence.is_some() { tx.commit()?; } - return Ok(replayed); + return Ok((replayed, echo_sequence)); } let terminal_is_live = tx .query_row( @@ -1068,7 +1076,10 @@ impl WorkspaceRegistry { advance_agent_hook_apply_cursor_transaction(&tx, sequence)?; } tx.commit()?; - Ok(ResourcePatchCommit { revision, result: result.clone(), replayed: false }) + Ok(( + ResourcePatchCommit { revision, result: result.clone(), replayed: false }, + echo_sequence, + )) } pub fn terminal_resource_id( From 6064c70c24219a729040b25825dfb1b59ab4f31d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:59:58 -0700 Subject: [PATCH 060/133] test(tui): cover retired roster fence compaction --- cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index fa01b3d92824..8c3d24f80f0c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -698,5 +698,14 @@ mod tests { assert_eq!(retired.retired_terminals.len(), RETIRED_TERMINAL_COUNT); assert_eq!(retired.retired_terminals.get("retired_0"), Some(&1)); assert_eq!(retired.retired_terminals.get("retired_2047"), Some(&4_095)); + + // Once the journal is checkpointed and sealed through sequence 2_047, + // the first half of these records is covered by the checkpoint and + // no longer needs an in-memory delayed-row fence. + assert!(retired.compact_retired_terminals(2_047)); + assert_eq!(retired.retired_terminals.len(), RETIRED_TERMINAL_COUNT / 2); + assert!(!retired.retired_terminals.contains_key("retired_1023")); + assert_eq!(retired.retired_terminals.get("retired_1024"), Some(&2_049)); + assert!(!retired.compact_retired_terminals(2_047)); } } From 30d1ebe4905c25d3637204658cdb444238f8ce13 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 08:02:14 -0700 Subject: [PATCH 061/133] fix(tui): compact retired roster fences at sealed checkpoints --- .../cmux-tui-core/src/journal_reducers.rs | 9 ++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 53 ++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 8c3d24f80f0c..1504b1413a94 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -342,6 +342,15 @@ impl AgentRoster { removed_entry || removed_fence || retired_changed } + /// Remove retirement fences whose source records are covered by a sealed + /// checkpoint. Segment sealing is checkpoint-aligned, so no reducer replay + /// can revisit a record at or below this watermark after compaction. + pub(crate) fn compact_retired_terminals(&mut self, through_sequence: u64) -> bool { + let before = self.retired_terminals.len(); + self.retired_terminals.retain(|_, retired_at| *retired_at > through_sequence); + self.retired_terminals.len() != before + } + pub(crate) fn is_retired(&self, terminal_id: &str) -> bool { self.retired_terminals.contains_key(terminal_id) } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 382a5858b89d..5626a66552dd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1135,7 +1135,19 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result 0 + && host.cursor >= sealed_through + && host.roster.compact_retired_terminals(sealed_through); + if host.cursor != initial_cursor || !snapshot_valid || compacted { registry.put_journal_reducer_state( AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, @@ -5743,6 +5755,39 @@ impl Mux { } } + /// Compact exact retirement fences after a checkpoint-aligned journal + /// segment seal. The fold lock serializes this with roster updates, and a + /// failed metadata write restores the previous in-memory snapshot so the + /// durable and live reducers never disagree. + fn compact_agent_roster_through(&self, through_sequence: u64) -> anyhow::Result<()> { + if through_sequence == 0 { + return Ok(()); + } + let _fold = self.agent_roster_fold.lock().unwrap(); + let (previous, cursor, snapshot) = { + let mut host = self.agent_roster.lock().unwrap(); + if host.cursor < through_sequence { + return Ok(()); + } + let previous = host.roster.clone(); + if !host.roster.compact_retired_terminals(through_sequence) { + return Ok(()); + } + (previous, host.cursor, host.roster.snapshot().to_string()) + }; + let persisted = self.workspace_registry.lock().unwrap().put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ); + if let Err(error) = persisted { + self.agent_roster.lock().unwrap().roster = previous; + return Err(error).context("persist compacted agent roster snapshot"); + } + Ok(()) + } + fn apply_roster_delta_to_record_cache(&self, delta: crate::journal_reducers::RosterDelta) { use crate::journal_reducers::RosterDelta; let mut records = self.agent_records.lock().unwrap(); @@ -6269,6 +6314,12 @@ impl Mux { if !commit.journal.replayed { self.publish_journal_event(); } + if let Err(error) = self.compact_agent_roster_through(commit.through_sequence) { + self.report_internal_diagnostic( + "journal sealed but agent roster compaction was deferred", + ); + eprintln!("cmux-tui: agent roster compaction failed: {error}"); + } return Ok(commit); } } From f0b81051ac2aaf597984dabd9048184f411e8b11 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 08:33:01 -0700 Subject: [PATCH 062/133] fix(tui): normalize journal echo commit tuple --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5626a66552dd..ab76dc1f0005 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9795,7 +9795,8 @@ impl Mux { &validated, "agent-report", key, - )? + ) + .map(|(commit, sequence)| (commit, Some(sequence)))? } (Some(sequence), _) => ( registry.commit_agent_projection_with_hook_state_and_sequence( From 56c81f3980e2e5b50be65af9dbb3f8849223e996 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 08:43:08 -0700 Subject: [PATCH 063/133] style(tui): apply rustfmt to agent projection match --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 29 ++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ab76dc1f0005..88dfe5bbc7f0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9783,20 +9783,21 @@ impl Mux { let (commit, echo_sequence) = match (journal_sequence, echo_ingress) { (None, Some((ingress, key))) => { let validated = self.journal_kernel.validate_ingress(ingress)?; - registry.commit_agent_projection_with_hook_state_and_journal( - mutation, - fingerprint, - expected_revision, - &terminal_id, - &value, - &deltas, - effective_hook_state, - ingress, - &validated, - "agent-report", - key, - ) - .map(|(commit, sequence)| (commit, Some(sequence)))? + registry + .commit_agent_projection_with_hook_state_and_journal( + mutation, + fingerprint, + expected_revision, + &terminal_id, + &value, + &deltas, + effective_hook_state, + ingress, + &validated, + "agent-report", + key, + ) + .map(|(commit, sequence)| (commit, Some(sequence)))? } (Some(sequence), _) => ( registry.commit_agent_projection_with_hook_state_and_sequence( From 9de15411b9e0adbc1025fe6b3fe7293ff7a3ef49 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 09:31:16 -0700 Subject: [PATCH 064/133] test(tui): seed agent rows through detected reports --- cmux-tui/crates/cmux-tui/src/app.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/app.rs b/cmux-tui/crates/cmux-tui/src/app.rs index c19fb5a55bc2..f64b35a667fc 100644 --- a/cmux-tui/crates/cmux-tui/src/app.rs +++ b/cmux-tui/crates/cmux-tui/src/app.rs @@ -41757,7 +41757,7 @@ mod tests { mux.report_agent( surface.id, AgentState::Working, - AgentSource::Hook, + AgentSource::Detected, Some("agent-session".into()), ) .unwrap(); @@ -41792,7 +41792,7 @@ mod tests { mux.report_agent( surface.id, AgentState::Working, - AgentSource::Hook, + AgentSource::Detected, Some("agent-session".into()), ) .unwrap(); @@ -41830,7 +41830,7 @@ mod tests { mux.report_agent( surface.id, AgentState::Done, - AgentSource::Hook, + AgentSource::Detected, Some("agent-session".into()), ) .unwrap(); @@ -41981,7 +41981,7 @@ mod tests { mux.report_agent( surface.id, AgentState::Working, - AgentSource::Hook, + AgentSource::Detected, Some("agent-session".into()), ) .unwrap(); From f2b48e7f20be80988229e5f637e109d257ed6604 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 09:35:43 -0700 Subject: [PATCH 065/133] test(tui): route core hook fixtures through journal --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 161 ++++++++++------------- 1 file changed, 73 insertions(+), 88 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 88dfe5bbc7f0..83ec2179549a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22465,6 +22465,7 @@ mod tests { fn agent_reports_apply_hook_authority() { let mux = test_mux(); let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); let events = mux.subscribe(); let initial_revision = mux.with_state(|state| state.resource_revision); let initial_epoch = mux.resource_event_epoch(); @@ -22492,14 +22493,15 @@ mod tests { && session.as_ref() == "socket-session" )); - let hook = mux - .report_agent( - surface.id, - AgentState::Blocked, - AgentSource::Hook, - Some("hook-session".to_string()), - ) - .unwrap(); + let hook_ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "PermissionRequest", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"hook-session"}), + ) + .unwrap(); + mux.append_journal_ingress(&hook_ingress, "test", "hook-authority").unwrap(); + let hook = mux.list_agents(Some(surface.id), None).remove(0); assert_eq!(hook.state, AgentState::Blocked); assert_eq!(hook.source, AgentSource::Hook); @@ -22516,7 +22518,7 @@ mod tests { let filtered = mux.list_agents(Some(surface.id), Some(AgentState::Blocked)); assert_eq!(filtered.len(), 1); - assert_eq!(filtered[0].session.as_deref(), Some("hook-session")); + assert_eq!(filtered[0].session, None); assert!(mux.list_agents(Some(surface.id), Some(AgentState::Done)).is_empty()); assert_eq!(mux.with_state(|state| state.resource_revision), initial_revision + 3); assert_eq!(mux.resource_event_epoch(), initial_epoch + 3); @@ -22529,7 +22531,7 @@ mod tests { assert_eq!(resource_events.batches[2].changes[0]["value"]["state"], "blocked"); assert_eq!( resource_events.batches[2].changes[0]["value"]["source_session"], - "hook-session" + Value::Null ); assert!(matches!( events.recv_timeout(Duration::from_millis(100)), @@ -22537,12 +22539,11 @@ mod tests { surface: event_surface, state, source, - session: Some(session), + session: None, .. }) if event_surface == surface.id && state.as_ref() == "blocked" && source.as_ref() == "hook" - && session.as_ref() == "hook-session" )); assert!(!events.try_iter().any(|event| matches!(event, MuxEvent::TreeChanged))); } @@ -22591,25 +22592,24 @@ mod tests { assert_eq!(raw.state, AgentState::Working); assert_eq!(mux.with_state(|state| state.resource_revision), created_revision + 1); - let hook_params = serde_json::json!({ - "machine":"current", - "session":"current", - "terminal_id":terminal_id, - "state":"blocked", - "source":"hook", - "source_session":"hook-session", - "expected_revision":(created_revision + 1).to_string(), - }); - let hook = public_request( - &mux, - "agent-hook", - "agent.report", - hook_params.clone(), - Some("agent-hook"), - ); - assert_eq!(hook["result"]["revision"], (created_revision + 2).to_string()); - assert_eq!(hook["result"]["value"]["source"], "hook"); - assert_eq!(hook["result"]["value"]["state"], "blocked"); + let hook_ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "PermissionRequest", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"hook-session"}), + ) + .unwrap(); + let hook_commit = + mux.append_journal_ingress(&hook_ingress, "test", "agent-hook").unwrap(); + assert!(!hook_commit.replayed); + assert_eq!(mux.with_state(|state| state.resource_revision), created_revision + 2); + let hook_value = crate::resource_api::public_session_snapshot(&mux).unwrap()["agents"] + .as_array() + .and_then(|agents| agents.first()) + .cloned() + .expect("hook projection"); + assert_eq!(hook_value["source"], "hook"); + assert_eq!(hook_value["state"], "blocked"); let ignored = mux .report_agent( @@ -22621,7 +22621,7 @@ mod tests { .unwrap(); assert_eq!(ignored.state, AgentState::Blocked); assert_eq!(ignored.source, AgentSource::Hook); - assert_eq!(ignored.session.as_deref(), Some("hook-session")); + assert_eq!(ignored.session, None); assert_eq!(mux.with_state(|state| state.resource_revision), created_revision + 3); assert_eq!(mux.resource_event_epoch(), initial_epoch + 3); assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 1); @@ -22633,11 +22633,12 @@ mod tests { ); assert_eq!(batches[0].changes[0]["value"]["source"], "socket"); assert_eq!(batches[0].changes[0]["value"]["source_session"], "raw-session"); - assert_eq!(batches[1].changes[0]["value"], hook["result"]["value"]); - assert_eq!(batches[2].changes[0]["value"], hook["result"]["value"]); + assert_eq!(batches[1].changes[0]["value"]["source"], "hook"); + assert_eq!(batches[1].changes[0]["value"]["state"], "blocked"); + assert_eq!(batches[2].changes[0]["value"], batches[1].changes[0]["value"]); assert_eq!( crate::resource_api::public_session_snapshot(&mux).unwrap()["agents"], - serde_json::json!([hook["result"]["value"].clone()]) + serde_json::json!([batches[1].changes[0]["value"].clone()]) ); mux.shutdown(); @@ -22659,22 +22660,17 @@ mod tests { ); assert_eq!( crate::resource_api::public_session_snapshot(&reopened).unwrap()["agents"], - serde_json::json!([hook["result"]["value"].clone()]) + serde_json::json!([hook_value.clone()]) ); let epoch_before_replay = reopened.resource_event_epoch(); let event_count_before_replay = reopened.resource_events_after(created_revision).unwrap().batches.len(); - let replay = public_request( - &reopened, - "agent-hook-replay", - "agent.report", - hook_params, - Some("agent-hook"), - ); - assert_eq!(replay["result"]["replayed"], true); - assert_eq!(replay["result"]["revision"], (created_revision + 2).to_string()); - assert_eq!(replay["result"]["value"], hook["result"]["value"]); + let replay = reopened + .append_journal_ingress(&hook_ingress, "test", "agent-hook") + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.sequence, hook_commit.sequence); assert_eq!(reopened.resource_event_epoch(), epoch_before_replay); assert_eq!( reopened.resource_events_after(created_revision).unwrap().batches.len(), @@ -22718,37 +22714,31 @@ mod tests { let hook_thread = { let mux = mux.clone(); let barrier = barrier.clone(); + let terminal_id = terminal_id.clone(); std::thread::spawn(move || { barrier.wait(); - mux.resource_report_agent_selected( - crate::ResourceSelectors { - machine: Some("current".into()), - session: Some("current".into()), - ..Default::default() - }, - &terminal_id, - AgentState::Blocked, - AgentSource::Hook, - Some("racing-hook".into()), - None, - &WorkspaceMutation::new("racing-hook", "resource-test").unwrap(), + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "PermissionRequest", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"racing-hook"}), ) - .unwrap() + .unwrap(); + mux.append_journal_ingress(&ingress, "test", "racing-hook").unwrap() }) }; barrier.wait(); let raw_result = raw_thread.join().unwrap(); let hook_commit = hook_thread.join().unwrap(); assert!(matches!(raw_result.source, AgentSource::Socket | AgentSource::Hook)); - assert!( - matches!(hook_commit.revision, value if value == revision + 1 || value == revision + 2) - ); + assert_eq!(mux.with_state(|state| state.resource_revision), revision + 2); + assert!(hook_commit.sequence > 0); let records = mux.list_agents(Some(surface_id), None); assert_eq!(records.len(), 1); assert_eq!(records[0].state, AgentState::Blocked); assert_eq!(records[0].source, AgentSource::Hook); - assert_eq!(records[0].session.as_deref(), Some("racing-hook")); + assert_eq!(records[0].session, None); assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 1); let batches = mux.resource_events_after(revision).unwrap().batches; assert_eq!(batches.len(), 2); @@ -23292,9 +23282,16 @@ mod tests { let mux = Mux::open_persistent("roster-hook-echo", SurfaceOptions::default(), &root).unwrap(); let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); - mux.report_agent(surface.id, AgentState::Working, AgentSource::Hook, Some("hook".into())) - .unwrap(); + let hook = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "SessionStart", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"hook"}), + ) + .unwrap(); + mux.append_journal_ingress(&hook, "test", "hook-owned-roster").unwrap(); let socket_echoes = || { mux.session_journal_after(0, 1024) .unwrap() @@ -24040,7 +24037,7 @@ mod tests { } #[test] - fn direct_hook_report_with_new_session_can_follow_hook_end() { + fn journal_hook_report_with_new_session_can_follow_hook_end() { let mux = test_mux(); let surface = mux.new_workspace(None, None).unwrap(); let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); @@ -24053,32 +24050,20 @@ mod tests { ) .unwrap() }; - mux.append_journal_ingress(&hook("SessionEnd", "old"), "test", "direct-hook-session-end") + mux.append_journal_ingress(&hook("SessionEnd", "old"), "test", "journal-hook-session-end") + .unwrap(); + mux.append_journal_ingress(&hook("SessionStart", "new"), "test", "journal-hook-session-start") .unwrap(); - assert!( - mux.report_agent( - surface.id, - AgentState::Working, - AgentSource::Hook, - Some("new".into()), - ) - .is_ok() - ); let record = &mux.list_agents(Some(surface.id), None)[0]; assert_eq!(record.source, AgentSource::Hook); - assert_eq!(record.session.as_deref(), Some("new")); - mux.apply_agent_hook_record(&hook("UserPromptSubmit", "new"), 2).unwrap(); + assert_eq!(record.session, None); + mux.append_journal_ingress(&hook("UserPromptSubmit", "new"), "test", "journal-hook-turn") + .unwrap(); assert_eq!(mux.agent_hook_fences.lock().unwrap()[&terminal_id].session_id, "new"); - assert_eq!(mux.agent_hook_fences.lock().unwrap()[&terminal_id].sequence, 2); - assert!( - mux.report_agent( - surface.id, - AgentState::Working, - AgentSource::Hook, - Some("old".into()), - ) - .is_err() - ); + assert_eq!(mux.agent_hook_fences.lock().unwrap()[&terminal_id].sequence, 3); + mux.append_journal_ingress(&hook("UserPromptSubmit", "old"), "test", "journal-hook-stale") + .unwrap(); + assert_eq!(mux.list_agents(Some(surface.id), None)[0].state, AgentState::Working); } #[test] From 634c94d7a95e531654f10c12811f0820512eea93 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 09:40:22 -0700 Subject: [PATCH 066/133] style(tui): format hook journal fixtures --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 83ec2179549a..93f259f1ac46 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22529,10 +22529,7 @@ mod tests { assert_eq!(resource_events.batches[1].changes[0]["value"]["source"], "hook"); assert_eq!(resource_events.batches[2].changes[0]["value"]["source"], "hook"); assert_eq!(resource_events.batches[2].changes[0]["value"]["state"], "blocked"); - assert_eq!( - resource_events.batches[2].changes[0]["value"]["source_session"], - Value::Null - ); + assert_eq!(resource_events.batches[2].changes[0]["value"]["source_session"], Value::Null); assert!(matches!( events.recv_timeout(Duration::from_millis(100)), Ok(MuxEvent::AgentChanged { @@ -22599,8 +22596,7 @@ mod tests { serde_json::json!({"session_id":"hook-session"}), ) .unwrap(); - let hook_commit = - mux.append_journal_ingress(&hook_ingress, "test", "agent-hook").unwrap(); + let hook_commit = mux.append_journal_ingress(&hook_ingress, "test", "agent-hook").unwrap(); assert!(!hook_commit.replayed); assert_eq!(mux.with_state(|state| state.resource_revision), created_revision + 2); let hook_value = crate::resource_api::public_session_snapshot(&mux).unwrap()["agents"] @@ -22666,9 +22662,7 @@ mod tests { let epoch_before_replay = reopened.resource_event_epoch(); let event_count_before_replay = reopened.resource_events_after(created_revision).unwrap().batches.len(); - let replay = reopened - .append_journal_ingress(&hook_ingress, "test", "agent-hook") - .unwrap(); + let replay = reopened.append_journal_ingress(&hook_ingress, "test", "agent-hook").unwrap(); assert!(replay.replayed); assert_eq!(replay.sequence, hook_commit.sequence); assert_eq!(reopened.resource_event_epoch(), epoch_before_replay); @@ -24052,8 +24046,12 @@ mod tests { }; mux.append_journal_ingress(&hook("SessionEnd", "old"), "test", "journal-hook-session-end") .unwrap(); - mux.append_journal_ingress(&hook("SessionStart", "new"), "test", "journal-hook-session-start") - .unwrap(); + mux.append_journal_ingress( + &hook("SessionStart", "new"), + "test", + "journal-hook-session-start", + ) + .unwrap(); let record = &mux.list_agents(Some(surface.id), None)[0]; assert_eq!(record.source, AgentSource::Hook); assert_eq!(record.session, None); From 253f1b6c28d3e84a869119949e305587da656017 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 09:47:00 -0700 Subject: [PATCH 067/133] test(tui): reserve socket adapter name --- cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs index 645076cccbae..3a68759fbb8d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs +++ b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs @@ -880,6 +880,13 @@ mod tests { assert_eq!(grok.kind, "agent.session.ended"); } + #[test] + fn socket_adapter_name_is_reserved_for_internal_reports() { + let error = + agent_hook_journal_ingress("socket", "PermissionRequest", None, json!({})).unwrap_err(); + assert!(error.to_string().contains("reserved for internal socket reports")); + } + #[test] fn provider_envelopes_normalize_structural_fields_and_redact_message_data() { let native = json!({ From e11dc8bdd6fa4c8ec7889edb8a265a5b7f74e713 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 09:47:15 -0700 Subject: [PATCH 068/133] fix(tui): reject reserved socket hook source --- cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs index 3a68759fbb8d..b73af2a51235 100644 --- a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs +++ b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs @@ -220,6 +220,10 @@ fn validate_agent_source(source: &str) -> anyhow::Result<()> { }), "agent source must contain 1 to {MAX_AGENT_SOURCE_BYTES} lowercase ASCII letters, digits, hyphens, or underscores" ); + anyhow::ensure!( + source != crate::journal_reducers::SOCKET_REPORT_ADAPTER, + "agent source is reserved for internal socket reports" + ); Ok(()) } From 502d249f6116cb0eec4e6b882642fd64493314c9 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:22:26 -0700 Subject: [PATCH 069/133] test(tui): recover terminal retirement after roster snapshot loss --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 59 ++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 93f259f1ac46..a3c50dd55f8b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23093,6 +23093,65 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn malformed_roster_snapshot_restores_terminal_retirement_fence() { + let root = std::env::temp_dir().join(format!( + "cmux-roster-retirement-recovery-{}", + crate::workspace_registry::new_uuid_v4() + )); + let session = "roster-retirement-recovery"; + let (terminal_id, delayed_sequence) = { + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"retirement-recovery"}), + ) + .unwrap(); + let validated = mux.journal_kernel.validate_ingress(&ingress).unwrap(); + let commit = mux + .workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&ingress, &validated, "test", "retirement-recovery") + .unwrap(); + + // Leave the journal row un-folded, then durably tombstone the + // terminal. A restart with a malformed reducer snapshot must + // still fence this delayed row. + close_terminal_runtime_for_test(&mux, &surface); + mux.shutdown(); + drop(mux); + (terminal_id, commit.sequence) + }; + + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + registry + .put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + 0, + "{malformed", + ) + .unwrap(); + let mut restored = restore_agent_roster(®istry).unwrap(); + assert!( + restored.roster.is_retired(terminal_id.as_str()), + "durable terminal tombstone must restore the roster fence" + ); + for record in registry.session_journal_after(0, 1024).unwrap().records { + restored.roster.apply(&crate::journal_reducers::RosterEvent::from_record(&record)); + } + assert!( + restored.roster.entries.is_empty(), + "delayed event at sequence {delayed_sequence} must remain fenced" + ); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn roster_restore_replays_beyond_legacy_record_cap() { let root = std::env::temp_dir() From 33bda6ca2f99bddd2463de90171a8b10054cb19a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:23:12 -0700 Subject: [PATCH 070/133] fix(tui): restore retirement fences from terminal tombstones --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 9 +++++++++ .../cmux-tui-core/src/workspace_registry.rs | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a3c50dd55f8b..0fdea6cf9007 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1135,6 +1135,15 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result anyhow::Result> { + let mut statement = self.connection.prepare( + "SELECT rt.public_id + FROM resource_terminals AS rt + JOIN terminal_hosts AS th ON th.terminal_id = rt.terminal_id + WHERE rt.lifecycle = 'tombstoned' OR th.lifecycle = 'tombstoned' + ORDER BY rt.deleted_revision ASC, rt.public_id ASC", + )?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .map(|row| Ok(TerminalPublicId::parse(row?)?)) + .collect() + } + pub fn replay_terminal( &self, mutation: &WorkspaceMutation, From fea0d3c0caa1c6e4a63bc5a9e384ad8790fb7a71 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:31:34 -0700 Subject: [PATCH 071/133] test(tui): fence retired ingress and detected reports --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 0fdea6cf9007..fb425f8c5b88 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23161,6 +23161,66 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn agent_hook_ingress_rejects_a_tombstoned_terminal() { + let root = std::env::temp_dir().join(format!( + "cmux-roster-retirement-ingress-{}", + crate::workspace_registry::new_uuid_v4() + )); + let session = "roster-retirement-ingress"; + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"after-retirement"}), + ) + .unwrap(); + let validated = mux.journal_kernel.validate_ingress(&ingress).unwrap(); + close_terminal_runtime_for_test(&mux, &surface); + + let error = mux + .workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&ingress, &validated, "test", "after-retirement") + .unwrap_err(); + assert!(error.to_string().contains("tombstoned"), "unexpected error: {error:#}"); + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn detected_report_cannot_replace_hook_owned_state() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let hook = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"hook-owned"}), + ) + .unwrap(); + mux.append_journal_ingress(&hook, "test", "hook-owned-detected").unwrap(); + + let record = mux + .report_agent( + surface.id, + AgentState::Idle, + AgentSource::Detected, + Some("screen-detected".into()), + ) + .unwrap(); + assert_eq!(record.source, AgentSource::Hook); + assert_eq!(record.state, AgentState::Working); + assert_eq!(record.agent.as_deref(), Some("claude")); + mux.shutdown(); + } + #[test] fn roster_restore_replays_beyond_legacy_record_cap() { let root = std::env::temp_dir() From 8f3cf1fdaf025af74aa7c405ba138e636c23dcec Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:34:22 -0700 Subject: [PATCH 072/133] fix(tui): enforce roster authority and coalesce snapshots --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 102 ++++++++++++++---- .../workspace_registry/journal_extensions.rs | 30 ++++++ 2 files changed, 112 insertions(+), 20 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fb425f8c5b88..e45badfb9565 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1094,8 +1094,15 @@ impl AgentSource { struct AgentRosterHost { roster: crate::journal_reducers::AgentRoster, cursor: u64, + /// Cursor included in the last durable snapshot. The journal remains the + /// source of truth between snapshots, so folds can coalesce metadata I/O. + persisted_cursor: u64, } +/// Bound synchronous reducer metadata writes while keeping restart replay +/// finite. Retirement and shutdown paths still force an immediate snapshot. +const AGENT_ROSTER_SNAPSHOT_INTERVAL: u64 = 32; + fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { use crate::journal_reducers::{ AGENT_ROSTER_REDUCER_ID, AGENT_ROSTER_REDUCER_VERSION, AgentRoster, RosterEvent, @@ -1110,7 +1117,7 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { snapshot_valid = true; - AgentRosterHost { roster, cursor } + AgentRosterHost { roster, cursor, persisted_cursor: cursor } } None => { eprintln!("cmux-tui: invalid agent roster snapshot; replaying journal"); @@ -1163,6 +1170,7 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result= AGENT_ROSTER_SNAPSHOT_INTERVAL); + should_persist.then(|| (host.cursor, host.roster.snapshot().to_string())) }; - if let Err(error) = - self.workspace_registry.lock().unwrap().put_journal_reducer_state( - crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, - crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, - cursor, - &snapshot, - ) - { - eprintln!("cmux-tui: persisting the agent roster snapshot failed: {error}"); + if let Some((cursor, snapshot)) = snapshot { + let persisted = + self.workspace_registry.lock().unwrap().put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + ); + match persisted { + Ok(()) => { + self.agent_roster.lock().unwrap().persisted_cursor = cursor; + } + Err(error) => { + eprintln!( + "cmux-tui: persisting the agent roster snapshot failed: {error}" + ); + } + } } } else { return; @@ -5794,6 +5824,7 @@ impl Mux { self.agent_roster.lock().unwrap().roster = previous; return Err(error).context("persist compacted agent roster snapshot"); } + self.agent_roster.lock().unwrap().persisted_cursor = cursor; Ok(()) } @@ -9734,16 +9765,17 @@ impl Mux { }); let now = reported_at_ms.unwrap_or_else(now_ms); let mut records = self.agent_records.lock().unwrap(); - let socket_report_ignored = records.get(&terminal_id).is_some_and(|existing| { + let lower_authority_report_ignored = records.get(&terminal_id).is_some_and(|existing| { existing.source == AgentSource::Hook - && source == AgentSource::Socket + && source != AgentSource::Hook && !effective_hook_state.is_some_and(|state| state.ended) }); - // Hook-owned state remains a semantic no-op for a socket report. A - // socket-owned report still refreshes its durable timestamp; the - // echo key below coalesces only when that complete payload is equal. + // Hook-owned state remains a semantic no-op for lower-authority + // detected and socket reports. A socket-owned report still refreshes + // its durable timestamp; the echo key below coalesces only when that + // complete payload is equal. let record = match records.get(&terminal_id) { - Some(existing) if socket_report_ignored => existing.clone(), + Some(existing) if lower_authority_report_ignored => existing.clone(), _ => TerminalAgentRecord { state: agent_state, source, @@ -9755,8 +9787,11 @@ impl Mux { // A socket report that is intentionally ignored by the hook-owned // record must persist that effective record, not the discarded socket // identity. Otherwise durable and in-memory projections diverge. - let persisted_source_session = - if socket_report_ignored { record.session.clone() } else { persisted_source_session }; + let persisted_source_session = if lower_authority_report_ignored { + record.session.clone() + } else { + persisted_source_session + }; let digest = Sha256::digest(format!("cmux.protocol/2/agent/{terminal_id}").as_bytes()); let payload = digest[..16].iter().map(|byte| format!("{byte:02x}")).collect::(); let agent_id = @@ -9948,6 +9983,7 @@ impl Mux { ); Some(error.context("persist agent roster tombstone")) } else { + self.agent_roster.lock().unwrap().persisted_cursor = cursor; None } } else { @@ -10039,12 +10075,38 @@ impl Mux { eprintln!("cmux-tui: journal hook workers did not stop before the shutdown deadline"); } self.finalize_terminal_journal("shutdown"); + if let Err(error) = self.persist_agent_roster_snapshot() { + eprintln!( + "cmux-tui: persisting the agent roster snapshot during shutdown failed: {error}" + ); + } self.journal_kernel.shutdown(); if let Some(runtime) = self.browser_runtime.lock().unwrap().take() { runtime.shutdown(); } } + /// Flush a coalesced roster cursor before the daemon exits. The journal is + /// already closed by the caller, so this write observes the final fold. + fn persist_agent_roster_snapshot(&self) -> anyhow::Result<()> { + let _fold = self.agent_roster_fold.lock().unwrap(); + let (cursor, snapshot) = { + let host = self.agent_roster.lock().unwrap(); + if host.cursor == host.persisted_cursor { + return Ok(()); + } + (host.cursor, host.roster.snapshot().to_string()) + }; + self.workspace_registry.lock().unwrap().put_journal_reducer_state( + crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, + crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, + cursor, + &snapshot, + )?; + self.agent_roster.lock().unwrap().persisted_cursor = cursor; + Ok(()) + } + fn finalize_terminal_journal(&self, context: &str) { if self.journal_ingress.is_closed() { if let Err(error) = self.journal_ingress.close_and_join() { diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs index 8e6eefe5e014..f3503da1d3e0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/journal_extensions.rs @@ -1482,6 +1482,9 @@ pub(crate) fn append_journal_ingress_transaction( subjects.insert(subject.clone()); } expand_topology_subjects(tx, &mut subjects)?; + if ingress.producer_id == crate::AGENT_HOOK_PRODUCER_ID { + ensure_agent_hook_terminals_are_live(tx, &ingress.subjects)?; + } let subjects = subjects.into_iter().collect::>(); let built_in_agent = ingress.producer_id == crate::AGENT_HOOK_PRODUCER_ID; let producer = JournalProducer { @@ -1551,6 +1554,33 @@ pub(crate) fn append_journal_ingress_transaction( Ok(JournalAppendCommit { sequence, event_id, replayed: false }) } +/// A terminal tombstone is final. Reject new hook rows after the close +/// transaction, including rows that were queued before the producer observed +/// the close. Rows already covered by an idempotency receipt return above and +/// remain replayable. +fn ensure_agent_hook_terminals_are_live( + tx: &Transaction<'_>, + subjects: &[JournalSubject], +) -> anyhow::Result<()> { + for subject in subjects.iter().filter(|subject| subject.kind == "terminal") { + let tombstoned = tx + .query_row( + "SELECT 1 + FROM resource_terminals AS rt + LEFT JOIN terminal_hosts AS th ON th.terminal_id = rt.terminal_id + WHERE (rt.public_id = ?1 OR th.terminal_id = ?1) + AND (rt.lifecycle = 'tombstoned' OR th.lifecycle = 'tombstoned') + LIMIT 1", + [subject.id.as_str()], + |_| Ok(()), + ) + .optional()? + .is_some(); + anyhow::ensure!(!tombstoned, "agent hook terminal {} is tombstoned", subject.id); + } + Ok(()) +} + impl WorkspaceRegistry { pub(crate) fn journal_hook_states(&self) -> anyhow::Result> { let mut statement = self.connection.prepare( From 64c360d93ea9344570548f24439efc10be5f1bf5 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:49:26 -0700 Subject: [PATCH 073/133] test(tui): coalesce repeated socket Done reports --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e45badfb9565..01b6df5e573c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23459,6 +23459,49 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn repeated_socket_done_reports_coalesce_after_roster_removal() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-done-echo-{}", crate::workspace_registry::new_uuid_v4())); + let mux = + Mux::open_persistent("roster-done-echo", SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let socket_echoes = || { + mux.session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count() + }; + + mux.report_agent(surface.id, AgentState::Done, AgentSource::Socket, Some("poll".into())) + .unwrap(); + assert_eq!(socket_echoes(), 1); + assert!(mux.agent_roster.lock().unwrap().roster.entries.is_empty()); + + mux.report_agent(surface.id, AgentState::Done, AgentSource::Socket, Some("poll".into())) + .unwrap(); + mux.report_agent(surface.id, AgentState::Done, AgentSource::Socket, Some("poll".into())) + .unwrap(); + assert_eq!( + socket_echoes(), + 1, + "repeated Done polls must reuse the last socket receipt after removal" + ); + + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn socket_reports_do_not_echo_when_hook_owns_roster_entry() { let root = std::env::temp_dir() From 600d95254531540830d8ede16f7e92d2f32d78e3 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:55:08 -0700 Subject: [PATCH 074/133] fix(tui): retain socket receipt after Done removal --- .../cmux-tui-core/src/journal_reducers.rs | 39 +++++++++++++++++-- cmux-tui/crates/cmux-tui-core/src/mux.rs | 30 ++++++++++---- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 1504b1413a94..52b8406e2389 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -30,8 +30,9 @@ pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; /// Version 2 added the agent adapter id to roster entries. Version 3 retains /// ended hook fences after their live roster entries are removed. Version 4 /// added a global authority bit. Version 5 removes that global bit and scopes -/// compatibility cleanup to each terminal's durable removal fence. -pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 5; +/// compatibility cleanup to each terminal's durable removal fence. Version 6 +/// retains the last socket semantic receipt after a Done entry is removed. +pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 6; /// Retirement tombstones protect delayed journal rows after a terminal leaves /// the resource tree. Keep each exact tombstone while its journal rows may be @@ -132,6 +133,12 @@ struct RosterFence { ended: bool, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct SocketReportReceipt { + state: String, + session: Option, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct RosterEntry { pub(crate) state: String, @@ -184,6 +191,12 @@ pub(crate) struct AgentRoster { /// retirement, so delayed records from that terminal cannot recreate it. #[serde(default)] retired_terminals: BTreeMap, + /// The last accepted socket state remains after a Done report removes + /// the live roster entry. This receipt is bounded by terminal lifecycle, + /// and lets repeated polls reuse the same semantic transition instead of + /// appending another journal row on every poll. + #[serde(default)] + socket_receipts: HashMap, } impl AgentRoster { @@ -270,6 +283,10 @@ impl AgentRoster { ended: state == AgentState::Done, }, ); + // A hook event claims this terminal's lifecycle. Any socket + // receipt from the previous lifecycle must not suppress a later + // socket transition after the hook state is gone. + self.socket_receipts.remove(terminal_id); (state, AgentSource::Hook, None, agent, event.committed_at_ms) }; if source == AgentSource::Socket @@ -282,6 +299,12 @@ impl AgentRoster { // overwrite it (mirrors the projection commit precedence). return Vec::new(); } + if source == AgentSource::Socket { + self.socket_receipts.insert( + terminal_id.to_string(), + SocketReportReceipt { state: state.as_str().to_string(), session: session.clone() }, + ); + } if state == AgentState::Done { // An ended agent leaves the roster entirely; the done state is // still committed to the durable projection by the host so @@ -324,6 +347,7 @@ impl AgentRoster { /// events yet, so the host retires entries explicitly. pub(crate) fn retire_terminal(&mut self, terminal_id: &str, retired_at: u64) -> bool { let removed_entry = self.entries.remove(terminal_id).is_some(); + let removed_socket_receipt = self.socket_receipts.remove(terminal_id).is_some(); // A retired terminal is no longer allowed to receive socket echoes, // and its ended hook fence is redundant with the retirement cursor. let removed_fence = self.hook_fences.remove(terminal_id).is_some(); @@ -339,7 +363,7 @@ impl AgentRoster { true } }; - removed_entry || removed_fence || retired_changed + removed_entry || removed_fence || removed_socket_receipt || retired_changed } /// Remove retirement fences whose source records are covered by a sealed @@ -383,6 +407,15 @@ impl AgentRoster { self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) } + /// Return the last accepted socket state for idempotency checks. The + /// receipt survives live-entry removal, but only semantic fields are + /// retained because report timestamps are intentionally non-semantic. + pub(crate) fn last_socket_report(&self, terminal_id: &str) -> Option<(&str, Option<&str>)> { + self.socket_receipts + .get(terminal_id) + .map(|receipt| (receipt.state.as_str(), receipt.session.as_deref())) + } + pub(crate) fn snapshot(&self) -> Value { serde_json::to_value(self).unwrap_or(Value::Null) } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 01b6df5e573c..7cce334fa818 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5872,14 +5872,28 @@ impl Mux { ) -> Option { let (cursor, previous) = { let host = self.agent_roster.lock().unwrap(); - let previous = host.roster.entries.get(terminal_id.as_str()).map(|entry| { - ( - entry.state.clone(), - entry.source.clone(), - entry.session.clone(), - entry.agent.clone(), - ) - }); + let previous = host + .roster + .entries + .get(terminal_id.as_str()) + .map(|entry| { + ( + entry.state.clone(), + entry.source.clone(), + entry.session.clone(), + entry.agent.clone(), + ) + }) + .or_else(|| { + host.roster.last_socket_report(terminal_id.as_str()).map(|(state, session)| { + ( + state.to_string(), + AgentSource::Socket.as_str().to_string(), + session.map(str::to_owned), + None, + ) + }) + }); (host.cursor, previous) }; let unchanged = previous.as_ref().is_some_and( From 1d5c678d0961325e6a44d52a6036edd9740b5353 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:20:49 -0700 Subject: [PATCH 075/133] test(tui): fence roster folds behind pending projections --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7cce334fa818..730890317667 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23806,6 +23806,55 @@ mod tests { assert_eq!(mux.list_agents(Some(surface.id), None)[0].state, AgentState::Working); } + #[test] + fn roster_fold_does_not_apply_hook_rows_still_pending_projection() { + let mux = test_mux(); + let first = mux.new_workspace(None, None).unwrap(); + let second = mux.new_workspace(None, None).unwrap(); + let first_terminal = first.terminal_public_id().cloned().expect("first terminal"); + let second_terminal = second.terminal_public_id().cloned().expect("second terminal"); + let ingress = |terminal_id: &TerminalPublicId, event: &str| { + crate::agent_hooks::agent_hook_journal_ingress( + "claude", + event, + Some(&terminal_id.to_string()), + serde_json::json!({}), + ) + .unwrap() + }; + + // The first journal row is durable, but its resource projection is + // deliberately unavailable. The next row can still commit for a + // different terminal. + mux.set_resource_patch_failure_for_test(true); + let first_receipt = mux + .append_journal_ingress( + &ingress(&first_terminal, "UserPromptSubmit"), + "test", + "pending-before-fold", + ) + .unwrap(); + mux.set_resource_patch_failure_for_test(false); + mux.append_journal_ingress( + &ingress(&second_terminal, "UserPromptSubmit"), + "test", + "fold-after-pending", + ) + .unwrap(); + + assert!( + mux.workspace_registry + .lock() + .unwrap() + .pending_agent_hook_projections_for_terminal(&first_terminal) + .unwrap() + .iter() + .any(|(_, _, _, sequence, _)| *sequence == first_receipt.sequence) + ); + assert!(mux.list_agents(Some(first.id), None).is_empty()); + assert_eq!(mux.list_agents(Some(second.id), None)[0].state, AgentState::Working); + } + #[test] fn journal_commit_preseeds_hook_retry_before_projection() { let mux = test_mux(); From d7754e8359a04d9b85adabe77431f1f106fbb38d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:23:15 -0700 Subject: [PATCH 076/133] test(tui): cover orphaned terminal tombstones --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 61 ++++++++++++++++--- .../src/workspace_registry/tests.rs | 30 +++++++++ 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 730890317667..651f1afd6b1c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2774,17 +2774,10 @@ impl Mux { pending: Vec<(String, String, String, u64, crate::JournalIngress)>, ) -> anyhow::Result { let mut applied = 0; + let mut fold_ingress = None; for (producer_id, origin, key, sequence, ingress) in pending { match self.apply_agent_hook_record(&ingress, sequence) { Ok(()) => { - self.fold_agent_roster( - &ingress, - &crate::JournalAppendCommit { - sequence, - event_id: String::new(), - replayed: true, - }, - ); if self .workspace_registry .lock() @@ -2793,6 +2786,11 @@ impl Mux { .is_ok() { applied += 1; + // Fold only after the pending marker is gone. Keep + // one representative ingress so a batch retry can + // catch the roster up through the current journal + // head in one pass. + fold_ingress = Some(ingress); } else { self.report_internal_diagnostic("agent hook retry cleanup deferred"); } @@ -2831,6 +2829,26 @@ impl Mux { } } } + if applied > 0 { + if let Some(ingress) = fold_ingress { + let head_sequence = self + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + .map(|page| page.head_sequence); + if let Ok(sequence) = head_sequence { + self.fold_agent_roster( + &ingress, + &crate::JournalAppendCommit { + sequence, + event_id: String::new(), + replayed: true, + }, + ); + } + } + } Ok(applied) } @@ -5712,6 +5730,20 @@ impl Mux { if commit.sequence <= read_cursor { return; } + let pending_sequences = match self + .workspace_registry + .lock() + .unwrap() + .pending_agent_hook_projection_sequences(read_cursor, commit.sequence) + { + Ok(sequences) => sequences, + Err(error) => { + eprintln!( + "cmux-tui: reading pending agent hook projections before roster fold failed: {error}" + ); + return; + } + }; loop { let page = match self .workspace_registry @@ -5735,6 +5767,13 @@ impl Mux { if record.sequence > commit.sequence { return; } + // The journal is durable before its resource projection. Do + // not advance the reducer across a pending row, because that + // would publish agent state that the authoritative projection + // rejected. A later retry will call this fold again. + if pending_sequences.contains(&record.sequence) { + return; + } let deltas = { let mut host = self.agent_roster.lock().unwrap(); let deltas = host.roster.apply(&RosterEvent::from_record(&record)); @@ -23852,6 +23891,12 @@ mod tests { .any(|(_, _, _, sequence, _)| *sequence == first_receipt.sequence) ); assert!(mux.list_agents(Some(first.id), None).is_empty()); + // The later projection is durable, but its roster fold waits behind + // the first unresolved row. Retrying the first row releases the + // contiguous prefix and catches both terminals up to the journal head. + assert!(mux.list_agents(Some(second.id), None).is_empty()); + mux.retry_pending_agent_hooks_for_terminal(&first_terminal).unwrap(); + assert_eq!(mux.list_agents(Some(first.id), None)[0].state, AgentState::Working); assert_eq!(mux.list_agents(Some(second.id), None)[0].state, AgentState::Working); } diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs index 7560d7daa151..1c5cc258ad1b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs @@ -3799,6 +3799,36 @@ fn terminal_close_tombstones_before_kill_and_retries_safely() { ); } +#[test] +fn tombstoned_terminal_ids_include_resource_rows_without_host_rows() { + let mut registry = WorkspaceRegistry::in_memory("test").unwrap(); + commit_terminal_topology(&mut registry, "orphaned-tombstone"); + let public_id = terminal_resource(TERMINAL_ONE); + + // Model a recovered database where the public resource tombstone survived + // but its legacy host row was lost. The query must still rebuild the + // terminal lifecycle fence from the resource table. + registry + .connection + .execute( + "UPDATE resource_terminals + SET lifecycle = 'tombstoned', updated_revision = 2, deleted_revision = 2 + WHERE public_id = ?1", + [public_id.as_str()], + ) + .unwrap(); + registry + .connection + .execute_batch( + "PRAGMA foreign_keys=OFF; + DELETE FROM terminal_hosts WHERE terminal_id = '00000000000040008000000000000001'; + PRAGMA foreign_keys=ON;", + ) + .unwrap(); + + assert_eq!(registry.tombstoned_terminal_public_ids().unwrap(), vec![public_id]); +} + #[test] fn closing_workspace_detaches_views_without_tombstoning_terminal_hosts() { let mut registry = WorkspaceRegistry::in_memory("test").unwrap(); From aeaab8da4db06dd2edcbfb73c69e543f65b1a38e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:24:17 -0700 Subject: [PATCH 077/133] fix(tui): fence roster replay behind durable projection --- .../cmux-tui-core/src/workspace_registry.rs | 2 +- .../src/workspace_registry/resource_store.rs | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index 8c0ca3f48d42..f0669938f54d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -2880,7 +2880,7 @@ impl WorkspaceRegistry { let mut statement = self.connection.prepare( "SELECT rt.public_id FROM resource_terminals AS rt - JOIN terminal_hosts AS th ON th.terminal_id = rt.terminal_id + LEFT JOIN terminal_hosts AS th ON th.terminal_id = rt.terminal_id WHERE rt.lifecycle = 'tombstoned' OR th.lifecycle = 'tombstoned' ORDER BY rt.deleted_revision ASC, rt.public_id ASC", )?; diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index cde19d206dd8..ac075e12b2ad 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -720,6 +720,37 @@ impl WorkspaceRegistry { .collect() } + /// Return pending hook journal sequences in a reducer replay range. + /// + /// A pending row means the durable journal event exists but its resource + /// projection has not been confirmed. Reducers must stop before that + /// sequence instead of folding later rows and creating a split-brain + /// projection. + pub(crate) fn pending_agent_hook_projection_sequences( + &self, + after_sequence: u64, + through_sequence: u64, + ) -> anyhow::Result> { + if after_sequence >= through_sequence { + return Ok(HashSet::new()); + } + let mut statement = self.connection.prepare( + "SELECT event_sequence + FROM resource_agent_hook_pending + WHERE event_sequence > ?1 AND event_sequence <= ?2", + )?; + statement + .query_map( + params![i64::try_from(after_sequence)?, i64::try_from(through_sequence)?,], + |row| row.get::<_, i64>(0), + )? + .map(|row| { + let sequence = row?; + u64::try_from(sequence).context("pending hook sequence is negative") + }) + .collect() + } + pub fn pending_agent_hook_projections_for_terminal( &self, terminal_id: &crate::resource::TerminalPublicId, From 5fcb670ef7a4db8aafe64d064fe34fbac4faec3b Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:28:49 -0700 Subject: [PATCH 078/133] test(tui): cap one roster fold pass --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 651f1afd6b1c..4dc803eba773 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23160,6 +23160,12 @@ mod tests { } let target = target.expect("target commit"); + mux.fold_agent_roster(&event, &target); + let first_pass_cursor = mux.agent_roster.lock().unwrap().cursor; + assert!( + first_pass_cursor < target.sequence, + "a report must process only a bounded journal prefix" + ); mux.fold_agent_roster(&event, &target); let host = mux.agent_roster.lock().unwrap(); assert_eq!(host.cursor, target.sequence); From 84445faba7cf575fc934f5d8b706711d24c3e8bd Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:29:24 -0700 Subject: [PATCH 079/133] fix(tui): bound synchronous roster replay --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4dc803eba773..33594056a90d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1102,6 +1102,10 @@ struct AgentRosterHost { /// Bound synchronous reducer metadata writes while keeping restart replay /// finite. Retirement and shutdown paths still force an immediate snapshot. const AGENT_ROSTER_SNAPSHOT_INTERVAL: u64 = 32; +/// Maximum number of journal rows one synchronous report may reduce. A stale +/// reducer cursor must not turn a polling request into an unbounded replay. +/// Later journal activity or a retry call continues from the durable cursor. +const AGENT_ROSTER_FOLD_RECORD_LIMIT: usize = 512; fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { use crate::journal_reducers::{ @@ -5744,6 +5748,7 @@ impl Mux { return; } }; + let mut folded_records = 0usize; loop { let page = match self .workspace_registry @@ -5760,6 +5765,7 @@ impl Mux { let mut advanced = false; let mut semantic_change = false; let mut removal = false; + let mut reached_record_limit = false; for record in page.records { if record.sequence <= read_cursor { continue; @@ -5767,6 +5773,10 @@ impl Mux { if record.sequence > commit.sequence { return; } + if folded_records >= AGENT_ROSTER_FOLD_RECORD_LIMIT { + reached_record_limit = true; + break; + } // The journal is durable before its resource projection. Do // not advance the reducer across a pending row, because that // would publish agent state that the authoritative projection @@ -5791,9 +5801,14 @@ impl Mux { } read_cursor = record.sequence; advanced = true; + folded_records += 1; if read_cursor == commit.sequence { break; } + if folded_records >= AGENT_ROSTER_FOLD_RECORD_LIMIT { + reached_record_limit = true; + break; + } } if advanced { let snapshot = { @@ -5827,6 +5842,9 @@ impl Mux { } else { return; } + if reached_record_limit { + return; + } if read_cursor >= commit.sequence { return; } From 33e9382d3ae82cb60231b3c45ca261a1c1f82406 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:58:58 -0700 Subject: [PATCH 080/133] fix(tui): continue bounded roster replay safely --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 274 ++++++++++++++++-- .../src/workspace_registry/resource_store.rs | 57 ++-- 2 files changed, 287 insertions(+), 44 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 33594056a90d..2090b90e5c56 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1104,8 +1104,19 @@ struct AgentRosterHost { const AGENT_ROSTER_SNAPSHOT_INTERVAL: u64 = 32; /// Maximum number of journal rows one synchronous report may reduce. A stale /// reducer cursor must not turn a polling request into an unbounded replay. -/// Later journal activity or a retry call continues from the durable cursor. +/// A coalesced background worker continues from the durable cursor. const AGENT_ROSTER_FOLD_RECORD_LIMIT: usize = 512; +/// A blocked roster fold must not keep a worker thread alive forever when the +/// projection is waiting on an external retry. The timeout is only a wake-up +/// bound; SQLite remains the source of truth and no state is inferred from it. +const AGENT_ROSTER_FOLD_WORKER_WAIT: Duration = Duration::from_secs(1); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AgentRosterFoldProgress { + ReachedTarget, + MoreAvailable, + WaitingForProjection, +} fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { use crate::journal_reducers::{ @@ -2193,6 +2204,13 @@ pub struct Mux { /// Serialize journal folds so a later commit cannot advance the reducer /// cursor past an earlier committed record. agent_roster_fold: Mutex<()>, + /// A single coalesced worker drains replay work that exceeds the bounded + /// synchronous report budget. The journal remains the source of truth. + agent_roster_fold_worker_running: AtomicBool, + /// Weak self-reference used to start the reducer worker from `&self` + /// entrypoints without creating a reference cycle. It is initialized once + /// immediately after the owning `Arc` is constructed. + self_reference: OnceLock>, agent_records: Mutex>, agent_hook_fences: Mutex>, /// Nonterminal notifications remain placement-local. Terminal unread @@ -2611,6 +2629,8 @@ impl Mux { sidebar_plugin: Mutex::new(SidebarPluginRuntime::default()), agent_roster: Mutex::new(agent_roster), agent_roster_fold: Mutex::new(()), + agent_roster_fold_worker_running: AtomicBool::new(false), + self_reference: OnceLock::new(), agent_records: Mutex::new(agent_records), agent_hook_fences: Mutex::new(agent_hook_fences), placement_notifications: Mutex::new(HashMap::new()), @@ -2653,6 +2673,10 @@ impl Mux { test_surface_runtime, session, }); + assert!( + mux.self_reference.set(Arc::downgrade(&mux)).is_ok(), + "mux self-reference must be initialized exactly once" + ); crate::journal_ingress::start(&mux, journal_ingress_receiver)?; mux.materialize_interrupted_resource_workspaces()?; mux.materialize_restored_browsers(&contents)?; @@ -5717,10 +5741,20 @@ impl Mux { ingress: &crate::JournalIngress, commit: &crate::JournalAppendCommit, ) { - use crate::journal_reducers::RosterEvent; if ingress.producer_id != crate::agent_hooks::AGENT_HOOK_PRODUCER_ID { return; } + if self.fold_agent_roster_through(commit.sequence) != AgentRosterFoldProgress::ReachedTarget + { + self.schedule_agent_roster_fold_worker(); + } + } + + /// Reduce at most one bounded batch of committed journal rows. The caller + /// owns continuation scheduling so this method never performs unbounded + /// work on a report or socket request. + fn fold_agent_roster_through(&self, target_sequence: u64) -> AgentRosterFoldProgress { + use crate::journal_reducers::RosterEvent; // Journal callbacks can complete out of order. Serialize folds and // replay the contiguous committed prefix, so a later sequence never // advances the cursor past an earlier event that has not been folded. @@ -5731,21 +5765,21 @@ impl Mux { // number of terminals. let _fold = self.agent_roster_fold.lock().unwrap(); let mut read_cursor = self.agent_roster.lock().unwrap().cursor; - if commit.sequence <= read_cursor { - return; + if target_sequence <= read_cursor { + return AgentRosterFoldProgress::ReachedTarget; } - let pending_sequences = match self + let (pending_sequences, quarantined_sequences) = match self .workspace_registry .lock() .unwrap() - .pending_agent_hook_projection_sequences(read_cursor, commit.sequence) + .agent_hook_projection_sequence_states(read_cursor, target_sequence) { Ok(sequences) => sequences, Err(error) => { eprintln!( "cmux-tui: reading pending agent hook projections before roster fold failed: {error}" ); - return; + return AgentRosterFoldProgress::WaitingForProjection; } }; let mut folded_records = 0usize; @@ -5759,19 +5793,20 @@ impl Mux { Ok(page) => page, Err(error) => { eprintln!("cmux-tui: reading committed agent events back failed: {error}"); - return; + return AgentRosterFoldProgress::WaitingForProjection; } }; let mut advanced = false; let mut semantic_change = false; let mut removal = false; + let mut skipped_quarantined = false; let mut reached_record_limit = false; for record in page.records { if record.sequence <= read_cursor { continue; } - if record.sequence > commit.sequence { - return; + if record.sequence > target_sequence { + return AgentRosterFoldProgress::WaitingForProjection; } if folded_records >= AGENT_ROSTER_FOLD_RECORD_LIMIT { reached_record_limit = true; @@ -5782,7 +5817,26 @@ impl Mux { // would publish agent state that the authoritative projection // rejected. A later retry will call this fold again. if pending_sequences.contains(&record.sequence) { - return; + return AgentRosterFoldProgress::WaitingForProjection; + } + if quarantined_sequences.contains(&record.sequence) { + // A dead-letter projection is an explicit authority + // decision not to apply the resource mutation. Advance + // the reducer cursor without replaying that rejected + // event, then allow later durable records to converge. + self.agent_roster.lock().unwrap().cursor = record.sequence; + read_cursor = record.sequence; + advanced = true; + skipped_quarantined = true; + folded_records += 1; + if read_cursor == target_sequence { + break; + } + if folded_records >= AGENT_ROSTER_FOLD_RECORD_LIMIT { + reached_record_limit = true; + break; + } + continue; } let deltas = { let mut host = self.agent_roster.lock().unwrap(); @@ -5802,7 +5856,7 @@ impl Mux { read_cursor = record.sequence; advanced = true; folded_records += 1; - if read_cursor == commit.sequence { + if read_cursor == target_sequence { break; } if folded_records >= AGENT_ROSTER_FOLD_RECORD_LIMIT { @@ -5814,9 +5868,10 @@ impl Mux { let snapshot = { let host = self.agent_roster.lock().unwrap(); let distance = host.cursor.saturating_sub(host.persisted_cursor); - let should_persist = semantic_change + let should_persist = (semantic_change || skipped_quarantined) && (host.persisted_cursor == 0 || removal + || skipped_quarantined || distance >= AGENT_ROSTER_SNAPSHOT_INTERVAL); should_persist.then(|| (host.cursor, host.roster.snapshot().to_string())) }; @@ -5840,14 +5895,119 @@ impl Mux { } } } else { - return; + return AgentRosterFoldProgress::WaitingForProjection; } if reached_record_limit { - return; + return AgentRosterFoldProgress::MoreAvailable; + } + if read_cursor >= target_sequence { + return AgentRosterFoldProgress::ReachedTarget; } - if read_cursor >= commit.sequence { + } + } + + /// Start one detached reducer worker when a bounded synchronous fold did + /// not reach its target. The worker owns no durable state and is woken by + /// the journal epoch; every pass rereads SQLite from the reducer cursor. + fn schedule_agent_roster_fold_worker(&self) { + if self + .agent_roster_fold_worker_running + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; + } + + let Some(mux) = self.self_reference.get().cloned() else { + self.agent_roster_fold_worker_running.store(false, Ordering::Release); + return; + }; + if let Err(error) = std::thread::Builder::new() + .name("agent-roster-fold".into()) + .spawn(move || Self::run_agent_roster_fold_worker(mux)) + { + self.agent_roster_fold_worker_running.store(false, Ordering::Release); + eprintln!("cmux-tui: could not start agent roster fold worker: {error}"); + } + } + + fn run_agent_roster_fold_worker(mux: Weak) { + loop { + let Some(mux) = mux.upgrade() else { return }; + if mux.shutting_down.load(Ordering::Acquire) { + mux.agent_roster_fold_worker_running.store(false, Ordering::Release); return; } + + let target_sequence = match mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + { + Ok(page) => page.head_sequence, + Err(error) => { + eprintln!( + "cmux-tui: reading the agent roster journal head in the worker failed: {error}" + ); + let epoch = mux.journal_event_epoch(); + mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_WAIT); + continue; + } + }; + + match mux.fold_agent_roster_through(target_sequence) { + AgentRosterFoldProgress::MoreAvailable => { + // Keep each pass bounded and yield between pages so a + // large restored journal cannot monopolize a core. + std::thread::yield_now(); + } + AgentRosterFoldProgress::WaitingForProjection => { + // A retry may clear the pending projection without + // appending a journal row. The bounded timeout handles + // that case while the epoch handles normal commits. + let epoch = mux.journal_event_epoch(); + mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_WAIT); + } + AgentRosterFoldProgress::ReachedTarget => { + // Close the worker only after a race-safe epoch/head + // handshake. A commit that arrives before the flag is + // cleared is either visible in the head or changes the + // epoch and causes this worker to retain ownership. + let observed_epoch = mux.journal_event_epoch(); + let current_head = match mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + { + Ok(page) => page.head_sequence, + Err(error) => { + eprintln!( + "cmux-tui: rechecking the agent roster journal head failed: {error}" + ); + let epoch = mux.journal_event_epoch(); + mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_WAIT); + continue; + } + }; + let current_cursor = mux.agent_roster.lock().unwrap().cursor; + if current_cursor < current_head { + continue; + } + + mux.agent_roster_fold_worker_running.store(false, Ordering::Release); + if mux.journal_event_epoch() != observed_epoch + && mux + .agent_roster_fold_worker_running + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + continue; + } + return; + } + } } } @@ -10140,6 +10300,7 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); self.config_reload_changed.notify_all(); + self.journal_event_changed.notify_all(); self.journal_kernel.wake_waiters(); let hook_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; if !self.journal_hook_runtime.shutdown_until(hook_deadline) { @@ -18267,6 +18428,19 @@ mod tests { Mux::new_for_test("test", SurfaceOptions::default()) } + /// Direct registry appends stage a pending hook receipt by design. Tests + /// that exercise only journal replay use this helper to model the paired + /// projection having already committed, without invoking the live report + /// path a second time. + fn mark_agent_hook_projections_applied_for_test(mux: &Mux) { + let pending = + mux.workspace_registry.lock().unwrap().pending_agent_hook_projections().unwrap(); + let mut registry = mux.workspace_registry.lock().unwrap(); + for (producer_id, origin, idempotency_key, _, _) in pending { + registry.clear_agent_hook_pending(&producer_id, &origin, &idempotency_key).unwrap(); + } + } + /// A machine resume reconnects every hosted terminal at once. Checkpoint /// capture can lose its consistency race while those reconnects append to /// the journal, but the skipped optimization is recovered by replaying @@ -23074,6 +23248,7 @@ mod tests { .unwrap() }; + mark_agent_hook_projections_applied_for_test(&mux); mux.fold_agent_roster(&second_event, &second_commit); let host = mux.agent_roster.lock().unwrap(); assert_eq!(host.cursor, second_commit.sequence); @@ -23177,18 +23352,54 @@ mod tests { ); } let target = target.expect("target commit"); + mark_agent_hook_projections_applied_for_test(&mux); - mux.fold_agent_roster(&event, &target); + assert_eq!( + mux.fold_agent_roster_through(target.sequence), + AgentRosterFoldProgress::MoreAvailable, + "the direct reducer pass must stop at the synchronous budget" + ); let first_pass_cursor = mux.agent_roster.lock().unwrap().cursor; assert!( first_pass_cursor < target.sequence, "a report must process only a bounded journal prefix" ); - mux.fold_agent_roster(&event, &target); + assert_eq!( + mux.fold_agent_roster_through(target.sequence), + AgentRosterFoldProgress::ReachedTarget + ); let host = mux.agent_roster.lock().unwrap(); assert_eq!(host.cursor, target.sequence); assert!(host.roster.entries.contains_key(terminal_id.as_str())); drop(host); + + // A later report must hand off a second oversized backlog to the + // coalesced worker. This proves that the synchronous cap does not + // leave the durable cursor permanently stale. + let mut worker_target = target; + for index in 513..1026 { + worker_target = mux + .workspace_registry + .lock() + .unwrap() + .append_journal_ingress( + &event, + &validated, + "test", + &format!("roster-multiple-pages-worker-{index}"), + ) + .unwrap(); + } + mark_agent_hook_projections_applied_for_test(&mux); + mux.fold_agent_roster(&event, &worker_target); + let worker_deadline = Instant::now() + Duration::from_secs(2); + while mux.agent_roster.lock().unwrap().cursor < worker_target.sequence { + assert!( + Instant::now() < worker_deadline, + "bounded roster fold worker did not drain the durable backlog" + ); + std::thread::yield_now(); + } mux.shutdown(); drop(mux); std::fs::remove_dir_all(root).unwrap(); @@ -23889,7 +24100,7 @@ mod tests { // The first journal row is durable, but its resource projection is // deliberately unavailable. The next row can still commit for a // different terminal. - mux.set_resource_patch_failure_for_test(true); + mux.set_resource_patch_failures_remaining_for_test(1); let first_receipt = mux .append_journal_ingress( &ingress(&first_terminal, "UserPromptSubmit"), @@ -23897,7 +24108,7 @@ mod tests { "pending-before-fold", ) .unwrap(); - mux.set_resource_patch_failure_for_test(false); + mux.set_resource_patch_failures_remaining_for_test(0); mux.append_journal_ingress( &ingress(&second_terminal, "UserPromptSubmit"), "test", @@ -23984,9 +24195,9 @@ mod tests { serde_json::json!({}), ) .unwrap(); - mux.workspace_registry.lock().unwrap().set_resource_patch_failure(true).unwrap(); + mux.set_resource_patch_failures_remaining_for_test(1); mux.append_journal_ingress(&ingress, "test", "hook-wake").unwrap(); - mux.workspace_registry.lock().unwrap().set_resource_patch_failure(false).unwrap(); + mux.set_resource_patch_failures_remaining_for_test(0); mux.report_agent(surface.id, AgentState::Working, AgentSource::Socket, None).unwrap(); assert!( @@ -24194,13 +24405,15 @@ mod tests { serde_json::json!({}), ) .unwrap(); - mux.workspace_registry.lock().unwrap().set_resource_patch_failure(true).unwrap(); + mux.set_resource_patch_failures_remaining_for_test( + crate::workspace_registry::AGENT_HOOK_MAX_ATTEMPTS as u64 + 1, + ); let receipt = mux.append_journal_ingress(&ingress, "test", "dead-letter").unwrap(); assert!(receipt.sequence > 0); for _ in 0..crate::workspace_registry::AGENT_HOOK_MAX_RETRY_PAGES_PER_WAKE { mux.retry_pending_agent_hooks_for_terminal(&terminal_id).unwrap(); } - mux.workspace_registry.lock().unwrap().set_resource_patch_failure(false).unwrap(); + mux.set_resource_patch_failures_remaining_for_test(0); let retry_state = mux .workspace_registry @@ -24227,6 +24440,19 @@ mod tests { mux.workspace_registry.lock().unwrap().pending_agent_hook_projections().unwrap().len(), 1 ); + + // A quarantined row must be skipped by the reducer. It must not hold + // every later, valid journal event behind a permanent retry fence. + let later = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"later": true}), + ) + .unwrap(); + let later_commit = mux.append_journal_ingress(&later, "test", "after-dead-letter").unwrap(); + assert_eq!(mux.agent_roster.lock().unwrap().cursor, later_commit.sequence); + assert!(mux.agent_roster.lock().unwrap().roster.entries.contains_key(terminal_id.as_str())); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index ac075e12b2ad..2c2e5d7b5977 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -720,35 +720,52 @@ impl WorkspaceRegistry { .collect() } - /// Return pending hook journal sequences in a reducer replay range. - /// - /// A pending row means the durable journal event exists but its resource - /// projection has not been confirmed. Reducers must stop before that - /// sequence instead of folding later rows and creating a split-brain - /// projection. - pub(crate) fn pending_agent_hook_projection_sequences( + /// Return live and quarantined hook journal sequences in a reducer replay + /// range. A live pending row means the durable journal event exists but + /// its resource projection has not been confirmed, so reducers must stop + /// before that sequence. A quarantined row exhausted its retry budget and + /// is intentionally skipped, otherwise it would permanently block every + /// later reducer record. + pub(crate) fn agent_hook_projection_sequence_states( &self, after_sequence: u64, through_sequence: u64, - ) -> anyhow::Result> { + ) -> anyhow::Result<(HashSet, HashSet)> { if after_sequence >= through_sequence { - return Ok(HashSet::new()); + return Ok((HashSet::new(), HashSet::new())); } let mut statement = self.connection.prepare( - "SELECT event_sequence + "SELECT event_sequence, attempt FROM resource_agent_hook_pending WHERE event_sequence > ?1 AND event_sequence <= ?2", )?; - statement - .query_map( - params![i64::try_from(after_sequence)?, i64::try_from(through_sequence)?,], - |row| row.get::<_, i64>(0), - )? - .map(|row| { - let sequence = row?; - u64::try_from(sequence).context("pending hook sequence is negative") - }) - .collect() + let mut pending = HashSet::new(); + let mut quarantined = HashSet::new(); + for row in statement.query_map( + params![i64::try_from(after_sequence)?, i64::try_from(through_sequence)?], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + )? { + let (sequence, attempt) = row?; + let sequence = u64::try_from(sequence).context("pending hook sequence is negative")?; + if attempt >= AGENT_HOOK_MAX_ATTEMPTS { + quarantined.insert(sequence); + } else { + pending.insert(sequence); + } + } + Ok((pending, quarantined)) + } + + /// Return live pending hook journal sequences in a reducer replay range. + /// Quarantined rows are excluded because they are handled as explicit + /// reducer skips by `agent_hook_projection_sequence_states`. + pub(crate) fn pending_agent_hook_projection_sequences( + &self, + after_sequence: u64, + through_sequence: u64, + ) -> anyhow::Result> { + self.agent_hook_projection_sequence_states(after_sequence, through_sequence) + .map(|(pending, _)| pending) } pub fn pending_agent_hook_projections_for_terminal( From 26d58910b109e350d0f9ab0b7caf5e282a113fef Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:14:14 -0700 Subject: [PATCH 081/133] fix(tui): serialize roster quarantine replay --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 192 ++++++++++++++---- .../src/workspace_registry/resource_store.rs | 39 +++- 2 files changed, 185 insertions(+), 46 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 2090b90e5c56..71bfacbd5b48 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1144,16 +1144,33 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result, ) -> anyhow::Result { + // Keep projection completion and removal of its pending marker in the + // same serialization domain as roster folding. This prevents a fold + // from observing a dead-letter marker while a recovery clears it. + let _fold = self.agent_roster_fold.lock().unwrap(); let mut applied = 0; let mut fold_ingress = None; for (producer_id, origin, key, sequence, ingress) in pending { + // The page was selected before this lock was acquired. Another + // retry can therefore have exhausted this receipt in the + // meantime. Recheck under the fold lock before applying or + // clearing anything, otherwise a stale successful retry could + // erase a durable dead-letter skip marker. + if self.workspace_registry.lock().unwrap().agent_hook_pending_is_quarantined( + &producer_id, + &origin, + &key, + )? { + continue; + } match self.apply_agent_hook_record(&ingress, sequence) { Ok(()) => { if self @@ -2857,6 +2890,7 @@ impl Mux { } } } + drop(_fold); if applied > 0 { if let Some(ingress) = fold_ingress { let head_sequence = self @@ -5672,58 +5706,77 @@ impl Mux { // a no-op for already-applied events while allowing restart repair. let lifecycle_event = agent_state_for_hook_kind(&ingress.kind).is_some(); let mut should_fold_roster = !lifecycle_event; - if let Err(error) = self.apply_agent_hook_record(ingress, commit.sequence) { - should_fold_roster = false; - if agent_hook_terminal_gone(&error) { + { + // The pending/quarantine decision, projection, and receipt cleanup + // are one serialized operation. The reducer takes this same lock + // while reading its status sets, so it cannot observe a live row + // being cleared or a dead letter being reclassified mid-fold. + let _fold = self.agent_roster_fold.lock().unwrap(); + let quarantined = self + .workspace_registry + .lock() + .unwrap() + .agent_hook_pending_is_quarantined(&ingress.producer_id, origin, idempotency_key)?; + if quarantined { + // A replay of a quarantined commit is an explicit no-op. Keep + // the receipt durable so the reducer can apply its skip rule. + should_fold_roster = true; + } else if let Err(error) = self.apply_agent_hook_record(ingress, commit.sequence) { + should_fold_roster = false; + if agent_hook_terminal_gone(&error) { + if let Err(_bookkeeping_error) = self + .workspace_registry + .lock() + .unwrap() + .clear_agent_hook_pending(&ingress.producer_id, origin, idempotency_key) + { + self.report_internal_diagnostic( + "terminal-gone agent hook cleanup deferred", + ); + } + } else { + if let Err(_bookkeeping_error) = + self.workspace_registry.lock().unwrap().enqueue_agent_hook_pending( + &ingress.producer_id, + origin, + idempotency_key, + commit.sequence, + ingress, + AGENT_HOOK_RETRY_ERROR, + agent_hook_retry_class(&error), + ) + { + self.report_internal_diagnostic( + "durable agent hook receipt remains staged after retry bookkeeping failure", + ); + } + } + } else { + if lifecycle_event { + // A stale or fenced hook can be accepted as a durable + // journal replay without advancing the projection + // watermark. Fold it only when this commit was the + // projection that advanced the watermark, so the roster + // cannot resurrect old state. + should_fold_roster = self + .workspace_registry + .lock() + .unwrap() + .agent_hook_apply_cursor() + .ok() + .is_some_and(|cursor| cursor == commit.sequence); + } if let Err(_bookkeeping_error) = self .workspace_registry .lock() .unwrap() .clear_agent_hook_pending(&ingress.producer_id, origin, idempotency_key) - { - self.report_internal_diagnostic("terminal-gone agent hook cleanup deferred"); - } - } else { - if let Err(_bookkeeping_error) = - self.workspace_registry.lock().unwrap().enqueue_agent_hook_pending( - &ingress.producer_id, - origin, - idempotency_key, - commit.sequence, - ingress, - AGENT_HOOK_RETRY_ERROR, - agent_hook_retry_class(&error), - ) { self.report_internal_diagnostic( - "durable agent hook receipt remains staged after retry bookkeeping failure", + "agent hook projection applied; retry bookkeeping cleanup deferred", ); } } - } else { - if lifecycle_event { - // A stale or fenced hook can be accepted as a durable journal - // replay without advancing the projection watermark. Fold it - // only when this commit was the projection that advanced the - // watermark, so the roster cannot resurrect old state. - should_fold_roster = self - .workspace_registry - .lock() - .unwrap() - .agent_hook_apply_cursor() - .ok() - .is_some_and(|cursor| cursor == commit.sequence); - } - if let Err(_bookkeeping_error) = self - .workspace_registry - .lock() - .unwrap() - .clear_agent_hook_pending(&ingress.producer_id, origin, idempotency_key) - { - self.report_internal_diagnostic( - "agent hook projection applied; retry bookkeeping cleanup deferred", - ); - } } if should_fold_roster { self.fold_agent_roster(ingress, &commit); @@ -24455,6 +24508,57 @@ mod tests { assert!(mux.agent_roster.lock().unwrap().roster.entries.contains_key(terminal_id.as_str())); } + #[test] + fn quarantined_hook_is_skipped_during_restart_replay() { + let root = std::env::temp_dir().join(format!( + "cmux-roster-quarantine-restart-{}", + crate::workspace_registry::new_uuid_v4() + )); + let session = "roster-quarantine-restart"; + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({}), + ) + .unwrap(); + mux.set_resource_patch_failures_remaining_for_test( + crate::workspace_registry::AGENT_HOOK_MAX_ATTEMPTS as u64 + 1, + ); + let commit = mux.append_journal_ingress(&ingress, "test", "restart-dead-letter").unwrap(); + for _ in 0..crate::workspace_registry::AGENT_HOOK_MAX_RETRY_PAGES_PER_WAKE { + mux.retry_pending_agent_hooks_for_terminal(&terminal_id).unwrap(); + } + mux.set_resource_patch_failures_remaining_for_test(0); + assert_eq!(mux.agent_roster.lock().unwrap().cursor, 0); + assert_eq!( + mux.workspace_registry + .lock() + .unwrap() + .agent_hook_pending_retry_state_for_test( + crate::agent_hooks::AGENT_HOOK_PRODUCER_ID, + "test", + "restart-dead-letter", + ) + .unwrap() + .expect("quarantined receipt") + .0, + crate::workspace_registry::AGENT_HOOK_MAX_ATTEMPTS + ); + mux.shutdown(); + drop(mux); + + let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + assert_eq!(reopened.agent_roster.lock().unwrap().cursor, commit.sequence); + assert!(reopened.agent_roster.lock().unwrap().roster.entries.is_empty()); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn sessionless_legacy_hook_identity_survives_restart() { let root = std::env::temp_dir() diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 2c2e5d7b5977..708391fd9059 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -626,11 +626,24 @@ impl WorkspaceRegistry { AGENT_HOOK_MAX_ATTEMPTS, ], )?; - // Keep quarantined failures bounded. Live retry rows remain untouched; - // only the oldest dead letters beyond the retention cap are evicted. + // Keep quarantined failures bounded once journal retention has removed + // their source records. Until then, every marker is needed to prevent + // an append-only journal replay from resurrecting the rejected event. + // Live retry rows remain untouched; only old dead letters beyond the + // retention cap and outside both active and sealed journal storage are + // evicted. self.connection.execute( "DELETE FROM resource_agent_hook_pending WHERE attempt >= ?1 + AND NOT EXISTS ( + SELECT 1 FROM session_journal + WHERE session_journal.sequence = resource_agent_hook_pending.event_sequence + ) + AND NOT EXISTS ( + SELECT 1 FROM journal_segments + WHERE journal_segments.start_sequence <= resource_agent_hook_pending.event_sequence + AND journal_segments.end_sequence >= resource_agent_hook_pending.event_sequence + ) AND rowid NOT IN ( SELECT rowid FROM resource_agent_hook_pending @@ -668,6 +681,28 @@ impl WorkspaceRegistry { Ok(()) } + /// Return whether a pending hook receipt has exhausted its retry budget. + /// Quarantined receipts are immutable dead letters. A replay must leave + /// them staged so the journal reducer can apply the explicit skip rule. + pub(crate) fn agent_hook_pending_is_quarantined( + &self, + producer_id: &str, + origin: &str, + idempotency_key: &str, + ) -> anyhow::Result { + self.connection + .query_row( + "SELECT attempt >= ?4 + FROM resource_agent_hook_pending + WHERE producer_id = ?1 AND origin = ?2 AND idempotency_key = ?3", + params![producer_id, origin, idempotency_key, AGENT_HOOK_MAX_ATTEMPTS], + |row| row.get::<_, bool>(0), + ) + .optional() + .map(|value| value.unwrap_or(false)) + .map_err(Into::into) + } + fn record_agent_hook_pending_failure( &self, producer_id: &str, From b96d664827802a232b4471b0f61a62c38882e2fe Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:39:37 -0700 Subject: [PATCH 082/133] fix(tui): wake roster replay after projection recovery --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 24 +++++++++++++++++-- .../src/workspace_registry/resource_store.rs | 6 +++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 71bfacbd5b48..e8a98f388654 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2865,6 +2865,11 @@ impl Mux { .is_ok() { applied += 1; + // Removing a pending marker can release a contiguous + // roster prefix even when the terminal is gone. Keep + // one representative ingress so the reducer is + // retried after the durable skip is recorded. + fold_ingress = Some(ingress); } else { self.report_internal_diagnostic("agent hook retry cleanup deferred"); } @@ -2892,6 +2897,10 @@ impl Mux { } drop(_fold); if applied > 0 { + // Pending-marker removal does not append a journal row. Wake a + // worker that is waiting on the journal epoch so it rereads the + // projection status immediately instead of relying on its timer. + self.wake_journal_event_waiters(); if let Some(ingress) = fold_ingress { let head_sequence = self .workspace_registry @@ -5377,6 +5386,10 @@ impl Mux { fn publish_journal_event(&self) { self.journal_kernel.notify_commit(); + self.wake_journal_event_waiters(); + } + + fn wake_journal_event_waiters(&self) { let mut epoch = self.journal_event_epoch.lock().unwrap(); *epoch = epoch.wrapping_add(1); self.journal_event_changed.notify_all(); @@ -23355,8 +23368,15 @@ mod tests { "cmux-roster-multiple-pages-{}", crate::workspace_registry::new_uuid_v4() )); - let mux = Mux::open_persistent("roster-multiple-pages", SurfaceOptions::default(), &root) - .unwrap(); + let registry = WorkspaceRegistry::open(&root, "roster-multiple-pages").unwrap(); + let mux = Mux::from_workspace_registry( + "roster-multiple-pages".into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); let surface = mux.new_workspace(None, None).unwrap(); let terminal_id = surface.terminal_public_id().cloned().expect("terminal"); let event = crate::JournalIngress { diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 708391fd9059..554cf6755022 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -1042,6 +1042,12 @@ impl WorkspaceRegistry { const OPERATION: &str = "agent.report"; validate_identifier("mutation id", &mutation.id)?; validate_identifier("mutation origin", &mutation.origin)?; + #[cfg(test)] + if self.resource_patch_failures_remaining.get() > 0 { + self.resource_patch_failures_remaining + .set(self.resource_patch_failures_remaining.get() - 1); + anyhow::bail!("forced one-shot resource patch failure"); + } anyhow::ensure!( result.get("terminal_id").and_then(Value::as_str) == Some(terminal_id.as_str()), "agent projection terminal does not match {terminal_id}" From f2795b9a0485c91537612bcd5d6f07fd093538be Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:50:19 -0700 Subject: [PATCH 083/133] test(tui): keep persistent roster replay coverage --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e8a98f388654..b63e0ae11f99 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2436,6 +2436,23 @@ impl Mux { ) } + #[cfg(test)] + fn open_persistent_for_test( + session: impl Into, + surface_options: SurfaceOptions, + state_root: &Path, + ) -> anyhow::Result> { + let session = session.into(); + let registry = WorkspaceRegistry::open(state_root, &session)?; + Self::from_workspace_registry( + session, + surface_options, + registry, + ProviderWorkspaceState::default(), + true, + ) + } + pub fn open_persistent_provider_managed( session: impl Into, surface_options: SurfaceOptions, @@ -23368,13 +23385,10 @@ mod tests { "cmux-roster-multiple-pages-{}", crate::workspace_registry::new_uuid_v4() )); - let registry = WorkspaceRegistry::open(&root, "roster-multiple-pages").unwrap(); - let mux = Mux::from_workspace_registry( - "roster-multiple-pages".into(), + let mux = Mux::open_persistent_for_test( + "roster-multiple-pages", SurfaceOptions::default(), - registry, - ProviderWorkspaceState::default(), - true, + &root, ) .unwrap(); let surface = mux.new_workspace(None, None).unwrap(); From bcd0cc3e445303d10fef473a682fc65c1854c46e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:58:44 -0700 Subject: [PATCH 084/133] test(tui): assert roster state behind pending projection --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b63e0ae11f99..b011a041d278 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24216,10 +24216,18 @@ mod tests { // The later projection is durable, but its roster fold waits behind // the first unresolved row. Retrying the first row releases the // contiguous prefix and catches both terminals up to the journal head. - assert!(mux.list_agents(Some(second.id), None).is_empty()); + { + let roster = mux.agent_roster.lock().unwrap(); + assert!(!roster.roster.entries.contains_key(second_terminal.as_str())); + } mux.retry_pending_agent_hooks_for_terminal(&first_terminal).unwrap(); assert_eq!(mux.list_agents(Some(first.id), None)[0].state, AgentState::Working); assert_eq!(mux.list_agents(Some(second.id), None)[0].state, AgentState::Working); + let roster = mux.agent_roster.lock().unwrap(); + assert_eq!( + roster.roster.entries[second_terminal.as_str()].agent_state(), + AgentState::Working + ); } #[test] From e10f28e6a7b863d3c04939ba5ff145e70be31223 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 14:44:59 -0700 Subject: [PATCH 085/133] fix(tui): keep roster retirement and socket coalescing durable --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 26 ++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b011a041d278..4e9f3217f786 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6170,7 +6170,7 @@ impl Mux { source: AgentSource, session: Option<&str>, ) -> Option { - let (cursor, previous) = { + let previous = { let host = self.agent_roster.lock().unwrap(); let previous = host .roster @@ -6194,7 +6194,7 @@ impl Mux { ) }) }); - (host.cursor, previous) + previous }; let unchanged = previous.as_ref().is_some_and( |(previous_state, previous_source, previous_session, _)| { @@ -6215,8 +6215,11 @@ impl Mux { format!("{state}|{source}|{:?}|{:?}", session, agent) }) .unwrap_or_default(); + // The key describes this terminal's semantic report only. A global + // journal cursor would make an unrelated event defeat coalescing and + // turn a steady socket poll into one durable row per event elsewhere. let material = format!( - "agent-report-echo-v2|{terminal_id}|{}|{}|{:?}|{cursor}|{previous}", + "agent-report-echo-v2|{terminal_id}|{}|{}|{:?}|{previous}", state.as_str(), source.as_str(), session, @@ -10199,7 +10202,14 @@ impl Mux { } state.resource_revision = commit.revision; if !commit.replayed { - records.insert(terminal_id.clone(), record.clone()); + if record.state == AgentState::Done { + // Done is a terminal state for the live compatibility view. + // The roster fold removes it, and a coalesced repeat has no + // fold callback, so remove it here as well. + records.remove(&terminal_id); + } else { + records.insert(terminal_id.clone(), record.clone()); + } } let echo_fold_sequence = echo_ingress.filter(|_| !commit.replayed).and_then(|_| echo_sequence); @@ -10271,10 +10281,10 @@ impl Mux { }; (journal_cursor, pending_cleanup) }; - if pending_cleanup.is_err() { + let pending_cleanup_error = pending_cleanup.err().map(|error| { self.report_internal_diagnostic("terminal agent hook cleanup deferred"); - } - pending_cleanup.context("purge pending agent hook receipts")?; + error.context("purge pending agent hook receipts") + }); let retired = { let mut host = self.agent_roster.lock().unwrap(); let previous = host.roster.clone(); @@ -10306,7 +10316,7 @@ impl Mux { self.agent_hook_fences.lock().unwrap().remove(terminal_id); self.agent_records.lock().unwrap().remove(terminal_id); self.terminal_notifications.lock().unwrap().remove(terminal_id); - snapshot_error.map_or(Ok(()), Err) + snapshot_error.or(pending_cleanup_error).map_or(Ok(()), Err) } fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) -> anyhow::Result<()> { From 118b8b46a6a735e11288f870343f4b9a0a9c9237 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:00:23 -0700 Subject: [PATCH 086/133] fix(cmux-tui): bound roster startup and serialize socket echoes --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 154 +++++++++++++++++++++-- 1 file changed, 144 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4e9f3217f786..2e590e61486c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1097,6 +1097,10 @@ struct AgentRosterHost { /// Cursor included in the last durable snapshot. The journal remains the /// source of truth between snapshots, so folds can coalesce metadata I/O. persisted_cursor: u64, + /// Startup replay stops after a bounded prefix. The owning `Mux` starts + /// the coalesced worker after its self-reference is ready when more rows + /// remain. + startup_replay_pending: bool, } /// Bound synchronous reducer metadata writes while keeping restart replay @@ -1106,6 +1110,9 @@ const AGENT_ROSTER_SNAPSHOT_INTERVAL: u64 = 32; /// reducer cursor must not turn a polling request into an unbounded replay. /// A coalesced background worker continues from the durable cursor. const AGENT_ROSTER_FOLD_RECORD_LIMIT: usize = 512; +/// Keep construction work bounded even when a snapshot is missing or stale. +/// The worker drains the remaining journal after the mux is usable. +const AGENT_ROSTER_STARTUP_FOLD_RECORD_LIMIT: usize = 512; /// A blocked roster fold must not keep a worker thread alive forever when the /// projection is waiting on an external retry. The timeout is only a wake-up /// bound; SQLite remains the source of truth and no state is inferred from it. @@ -1132,7 +1139,12 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result { snapshot_valid = true; - AgentRosterHost { roster, cursor, persisted_cursor: cursor } + AgentRosterHost { + roster, + cursor, + persisted_cursor: cursor, + startup_replay_pending: false, + } } None => { eprintln!("cmux-tui: invalid agent roster snapshot; replaying journal"); @@ -1143,8 +1155,17 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result AgentRosterHost::default(), }; let initial_cursor = host.cursor; + let mut replayed_records = 0usize; loop { - let page = registry.session_journal_after(host.cursor, AGENT_ROSTER_FOLD_RECORD_LIMIT)?; + let remaining_budget = + AGENT_ROSTER_STARTUP_FOLD_RECORD_LIMIT.saturating_sub(replayed_records); + if remaining_budget == 0 { + break; + } + let page = registry.session_journal_after( + host.cursor, + remaining_budget.min(AGENT_ROSTER_FOLD_RECORD_LIMIT), + )?; if page.records.is_empty() { break; } @@ -1152,6 +1173,7 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result anyhow::Result= page.head_sequence { + break; + } + if replayed_records >= AGENT_ROSTER_STARTUP_FOLD_RECORD_LIMIT { + host.startup_replay_pending = true; + break; + } } // Terminal retirement is durable in the resource registry, not in the // agent journal. Rebuild those fences after replay so a lost or malformed @@ -2735,6 +2767,13 @@ impl Mux { } mux.reconcile_retired_terminal_side_tables(); mux.retry_pending_agent_hooks()?; + let startup_replay_pending = mux.agent_roster.lock().unwrap().startup_replay_pending; + if startup_replay_pending { + // Construction replay is intentionally bounded. Start the worker + // only after the Arc self-reference exists and restored surfaces + // are materialized, so it can safely finish the durable tail. + mux.schedule_agent_roster_fold_worker(); + } crate::journal_hooks::start(&mux)?; Ok(mux) } @@ -9811,6 +9850,13 @@ impl Mux { "source":source.as_str(), "source_session":session, }); + // Socket echo admission and its resource commit must share the same + // fence. Otherwise two equal polls can both derive the same key + // before either one publishes the receipt, and the replay path would + // incorrectly report a missing fresh record. + let socket_sequence_guard = (source == AgentSource::Socket && !sequence_lock_held) + .then(|| self.agent_hook_fences.lock().unwrap()); + let commit_sequence_lock_held = sequence_lock_held || socket_sequence_guard.is_some(); let (echo_sequence, echo_timestamp, echo_ingress) = if source == AgentSource::Socket { let terminal_id = { let state_guard = self.state.lock().unwrap(); @@ -9844,7 +9890,7 @@ impl Mux { } else { (journal_sequence, None, None) }; - let (_, record) = self.commit_agent_report( + let commit_result = self.commit_agent_report( AgentReportTarget::Surface(surface), state, source, @@ -9852,14 +9898,16 @@ impl Mux { None, &mutation, &fingerprint, - sequence_lock_held, + commit_sequence_lock_held, hook_state.as_ref(), echo_sequence, origin, agent_adapter, echo_timestamp, echo_ingress.as_ref(), - )?; + ); + drop(socket_sequence_guard); + let (_, record) = commit_result?; let record = record.context("fresh raw agent report unexpectedly replayed")?; if source != AgentSource::Hook { // A successful report means this terminal is available. Retry only @@ -9888,6 +9936,11 @@ impl Mux { "source":source.as_str(), "source_session":source_session, }); + // Hold the hook fence while deriving the semantic echo key and + // committing it. This closes the equal-poll race without adding a + // second, independent serialization primitive. + let socket_sequence_guard = + (source == AgentSource::Socket).then(|| self.agent_hook_fences.lock().unwrap()); let (journal_sequence, reported_at_ms, echo_ingress) = if source == AgentSource::Socket { let timestamp = now_ms(); match self.agent_report_echo_key( @@ -9920,7 +9973,7 @@ impl Mux { expected_revision, mutation, &fingerprint, - false, + socket_sequence_guard.is_some(), None, journal_sequence, AgentReportOrigin::Direct, @@ -9928,6 +9981,7 @@ impl Mux { reported_at_ms, echo_ingress.as_ref(), ); + drop(socket_sequence_guard); if result.is_ok() && source != AgentSource::Hook { let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); } @@ -10216,6 +10270,17 @@ impl Mux { drop(records); drop(state); drop(registry); + // The fence protects the commit only. Release it before folding the + // journal row so a future fold implementation cannot re-enter the + // report serialization domain. + drop(sequence_guard); + let publish_echo = !commit.replayed && echo_sequence.is_some(); + if publish_echo { + // This transaction appended a journal row as well as updating the + // resource projection. Wake tail readers before the synchronous + // reducer work so they never wait for an unrelated event. + self.publish_journal_event(); + } if let (Some((ingress, _)), Some(sequence)) = (echo_ingress, echo_fold_sequence) { self.fold_agent_roster( ingress, @@ -10232,7 +10297,9 @@ impl Mux { updated_at_ms: record.updated_at_ms, }; if !commit.replayed { - self.publish_resource_event(); + if !publish_echo { + self.publish_resource_event(); + } self.emit(MuxEvent::AgentChanged { surface: agent.surface, state: Arc::from(agent.state.as_str()), @@ -23669,7 +23736,7 @@ mod tests { } #[test] - fn roster_restore_replays_beyond_legacy_record_cap() { + fn roster_restore_bounds_startup_replay_and_schedules_the_tail() { let root = std::env::temp_dir() .join(format!("cmux-roster-large-replay-{}", crate::workspace_registry::new_uuid_v4())); let mux = @@ -23684,7 +23751,7 @@ mod tests { ) .unwrap(); let validated = mux.journal_kernel.validate_ingress(&event).unwrap(); - for index in 0..100_001 { + for index in 0..(AGENT_ROSTER_STARTUP_FOLD_RECORD_LIMIT + 1) { mux.workspace_registry .lock() .unwrap() @@ -23708,9 +23775,30 @@ mod tests { "{malformed", ) .unwrap(); + let head = registry.session_journal_after(0, 1).unwrap().head_sequence; let restored = restore_agent_roster(®istry).unwrap(); - assert_eq!(restored.cursor, registry.session_journal_after(0, 1).unwrap().head_sequence); + assert!(restored.cursor < head, "startup replay must stop before the tail"); + assert!(restored.startup_replay_pending); assert!(restored.roster.entries.contains_key(terminal_id.as_str())); + + // Construction starts the bounded continuation only after the mux + // owns a self-reference. The worker must eventually converge without + // making startup itself scan the full journal. + let mux = Mux::from_workspace_registry( + "roster-large-replay".into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while mux.agent_roster.lock().unwrap().cursor < head { + assert!(Instant::now() < deadline, "roster worker did not drain the tail"); + std::thread::yield_now(); + } + mux.shutdown(); + drop(mux); std::fs::remove_dir_all(root).unwrap(); } @@ -23887,6 +23975,52 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn concurrent_identical_socket_reports_share_one_echo_receipt() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let left = mux.clone(); + let right = mux.clone(); + let surface_id = surface.id; + let (left_result, right_result) = std::thread::scope(|scope| { + let left = scope.spawn(|| { + left.report_agent( + surface_id, + AgentState::Working, + AgentSource::Socket, + Some("concurrent-poll".into()), + ) + }); + let right = scope.spawn(|| { + right.report_agent( + surface_id, + AgentState::Working, + AgentSource::Socket, + Some("concurrent-poll".into()), + ) + }); + (left.join().expect("left report thread"), right.join().expect("right report thread")) + }); + assert!(left_result.is_ok(), "left socket report failed: {left_result:?}"); + assert!(right_result.is_ok(), "right socket report failed: {right_result:?}"); + let echoes = mux + .session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count(); + assert_eq!(echoes, 1, "equal concurrent polls must append one echo"); + mux.shutdown(); + } + #[test] fn socket_reports_do_not_echo_when_hook_owns_roster_entry() { let root = std::env::temp_dir() From bc6e7367ca905b71581a6a9b65a114413773e6c1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:12:52 -0700 Subject: [PATCH 087/133] fix tui agent roster projection notifications --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 2e590e61486c..f11af4da4e51 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2573,18 +2573,20 @@ impl Mux { agent: entry.agent.clone(), updated_at_ms: entry.updated_at_ms, }; - // A repeated socket poll refreshes the compatibility projection + // A repeated direct report refreshes the compatibility projection // without appending a journal echo. On restart, retain that newer // timestamp when the roster snapshot still has the older echo. - let keep_newer_socket_record = + // Preserve only records with the same source, since source + // authority determines which projection is allowed to win. + let keep_newer_direct_record = agent_records.get(&terminal_id).is_some_and(|existing| { - existing.source == AgentSource::Socket - && record.source == AgentSource::Socket + matches!(existing.source, AgentSource::Socket | AgentSource::Detected) + && existing.source == record.source && existing.state == record.state && existing.session == record.session && existing.updated_at_ms > record.updated_at_ms }); - if !keep_newer_socket_record { + if !keep_newer_direct_record { agent_records.insert(terminal_id, record); } } @@ -10297,9 +10299,10 @@ impl Mux { updated_at_ms: record.updated_at_ms, }; if !commit.replayed { - if !publish_echo { - self.publish_resource_event(); - } + // Combined echo commits update both durable journal state and the + // resource projection. Publish both notifications so consumers of + // either stream observe the commit. + self.publish_resource_event(); self.emit(MuxEvent::AgentChanged { surface: agent.surface, state: Arc::from(agent.state.as_str()), From 3b2d153606391f8dc3f9ab0ef7183102899fb98c Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:22:38 -0700 Subject: [PATCH 088/133] fix(tui): release hook fences before roster folds --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 102 ++++++++++++++++++----- 1 file changed, 79 insertions(+), 23 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f11af4da4e51..74d58490df30 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1347,6 +1347,15 @@ enum AgentReportOrigin { RosterFold, } +/// The portion of an agent report that must be folded after all report +/// serialization locks are released. Keeping this receipt separate from the +/// resource transaction prevents a socket fence from being held while the +/// journal reducer takes its own lock. +struct AgentReportFoldReceipt { + ingress: crate::JournalIngress, + sequence: u64, +} + enum AgentReportTarget<'a> { Surface(SurfaceId), Resource { selectors: &'a crate::ResourceSelectors, terminal_id: &'a TerminalPublicId }, @@ -2876,9 +2885,10 @@ impl Mux { &self, pending: Vec<(String, String, String, u64, crate::JournalIngress)>, ) -> anyhow::Result { - // Keep projection completion and removal of its pending marker in the - // same serialization domain as roster folding. This prevents a fold - // from observing a dead-letter marker while a recovery clears it. + // Keep one lock order for retry admission and reducer bookkeeping. + // The hook fence is acquired first because applying a hook record + // serializes its lifecycle under that fence. + let mut fences = self.agent_hook_fences.lock().unwrap(); let _fold = self.agent_roster_fold.lock().unwrap(); let mut applied = 0; let mut fold_ingress = None; @@ -2895,7 +2905,7 @@ impl Mux { )? { continue; } - match self.apply_agent_hook_record(&ingress, sequence) { + match self.apply_agent_hook_record_locked(&ingress, sequence, &mut fences) { Ok(()) => { if self .workspace_registry @@ -2954,6 +2964,7 @@ impl Mux { } } drop(_fold); + drop(fences); if applied > 0 { // Pending-marker removal does not append a journal row. Wake a // worker that is waiting on the journal epoch so it rereads the @@ -5777,12 +5788,17 @@ impl Mux { // a no-op for already-applied events while allowing restart repair. let lifecycle_event = agent_state_for_hook_kind(&ingress.kind).is_some(); let mut should_fold_roster = !lifecycle_event; + // Keep one lock order for hook admission and reducer bookkeeping. + // Socket reports take the hook fence before their resource commit and + // fold only after releasing it. Hook ingress follows the same order, + // then releases both locks before starting the reducer fold. + let mut fences = self.agent_hook_fences.lock().unwrap(); + let _fold = self.agent_roster_fold.lock().unwrap(); { // The pending/quarantine decision, projection, and receipt cleanup // are one serialized operation. The reducer takes this same lock // while reading its status sets, so it cannot observe a live row // being cleared or a dead letter being reclassified mid-fold. - let _fold = self.agent_roster_fold.lock().unwrap(); let quarantined = self .workspace_registry .lock() @@ -5792,7 +5808,9 @@ impl Mux { // A replay of a quarantined commit is an explicit no-op. Keep // the receipt durable so the reducer can apply its skip rule. should_fold_roster = true; - } else if let Err(error) = self.apply_agent_hook_record(ingress, commit.sequence) { + } else if let Err(error) = + self.apply_agent_hook_record_locked(ingress, commit.sequence, &mut fences) + { should_fold_roster = false; if agent_hook_terminal_gone(&error) { if let Err(_bookkeeping_error) = self @@ -5849,6 +5867,8 @@ impl Mux { } } } + drop(_fold); + drop(fences); if should_fold_roster { self.fold_agent_roster(ingress, &commit); } @@ -6327,6 +6347,19 @@ impl Mux { &self, ingress: &crate::JournalIngress, sequence: u64, + ) -> anyhow::Result<()> { + let mut fences = self.agent_hook_fences.lock().unwrap(); + self.apply_agent_hook_record_locked(ingress, sequence, &mut fences) + } + + /// Apply a hook record while the caller owns the hook-fence lock. The + /// caller may also hold the roster-fold lock, so every such path must take + /// the locks in the order `agent_hook_fences` then `agent_roster_fold`. + fn apply_agent_hook_record_locked( + &self, + ingress: &crate::JournalIngress, + sequence: u64, + fences: &mut HashMap, ) -> anyhow::Result<()> { if ingress.producer_id != crate::agent_hooks::AGENT_HOOK_PRODUCER_ID { return Ok(()); @@ -6360,7 +6393,6 @@ impl Mux { // update as one operation. The projection path takes registry, state, // then agent-record locks, so teardown acquires sequence before those // locks as well. - let mut fences = self.agent_hook_fences.lock().unwrap(); let explicit_session_id = ingress .payload .get("normalized") @@ -6441,9 +6473,6 @@ impl Mux { terminal_id.clone(), HookFence { session_id: agent_session_id, sequence, ended: state == AgentState::Done }, ); - // Projection ordering is complete. Do not carry the fence guard into - // cleanup or any retry/reentrant path. - drop(fences); // An ended session leaves the roster entirely: the done state was // committed and broadcast above (so remote caches converge), and the // live record is dropped so agents views stop listing the terminal @@ -9909,7 +9938,17 @@ impl Mux { echo_ingress.as_ref(), ); drop(socket_sequence_guard); - let (_, record) = commit_result?; + let (_, record, fold_receipt) = commit_result?; + if let Some(receipt) = fold_receipt { + self.fold_agent_roster( + &receipt.ingress, + &crate::JournalAppendCommit { + sequence: receipt.sequence, + event_id: String::new(), + replayed: false, + }, + ); + } let record = record.context("fresh raw agent report unexpectedly replayed")?; if source != AgentSource::Hook { // A successful report means this terminal is available. Retry only @@ -9987,7 +10026,22 @@ impl Mux { if result.is_ok() && source != AgentSource::Hook { let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); } - result.map(|(commit, _)| commit) + match result { + Ok((commit, _, fold_receipt)) => { + if let Some(receipt) = fold_receipt { + self.fold_agent_roster( + &receipt.ingress, + &crate::JournalAppendCommit { + sequence: receipt.sequence, + event_id: String::new(), + replayed: false, + }, + ); + } + Ok(commit) + } + Err(error) => Err(error), + } } #[allow(clippy::too_many_arguments)] @@ -10007,7 +10061,8 @@ impl Mux { agent_adapter: Option, reported_at_ms: Option, echo_ingress: Option<&(crate::JournalIngress, String)>, - ) -> anyhow::Result<(ResourcePatchCommit, Option)> { + ) -> anyhow::Result<(ResourcePatchCommit, Option, Option)> + { // Hook state is accepted only through the validated journal ingress. // A direct Hook report has no durable event to replay, so allowing it // would let a restart overwrite the journal-derived projection. @@ -10029,7 +10084,7 @@ impl Mux { // observed. Repair the durable apply watermark separately. registry.advance_agent_hook_apply_cursor(sequence)?; } - return Ok((replay, None)); + return Ok((replay, None, None)); } let mut state = self.state.lock().unwrap(); let (surface, terminal_id) = match target { @@ -10267,8 +10322,10 @@ impl Mux { records.insert(terminal_id.clone(), record.clone()); } } - let echo_fold_sequence = - echo_ingress.filter(|_| !commit.replayed).and_then(|_| echo_sequence); + let fold_receipt = echo_ingress.filter(|_| !commit.replayed).and_then(|(ingress, _)| { + echo_sequence + .map(|sequence| AgentReportFoldReceipt { ingress: ingress.clone(), sequence }) + }); drop(records); drop(state); drop(registry); @@ -10283,12 +10340,6 @@ impl Mux { // reducer work so they never wait for an unrelated event. self.publish_journal_event(); } - if let (Some((ingress, _)), Some(sequence)) = (echo_ingress, echo_fold_sequence) { - self.fold_agent_roster( - ingress, - &crate::JournalAppendCommit { sequence, event_id: String::new(), replayed: false }, - ); - } let agent = AgentRecord { surface, terminal_id, @@ -10312,7 +10363,7 @@ impl Mux { updated_at_ms: agent.updated_at_ms, }); } - Ok((commit, Some(agent))) + Ok((commit, Some(agent), fold_receipt)) } /// Drop per-surface metadata for a surface that has left the tree. @@ -10383,6 +10434,11 @@ impl Mux { } else { None }; + // The fold lock protects the durable tombstone and snapshot only. + // Release it before touching the hook-fence table so teardown never + // takes `agent_roster_fold` then `agent_hook_fences` while ingress + // takes the inverse order. + drop(_fold); self.agent_hook_fences.lock().unwrap().remove(terminal_id); self.agent_records.lock().unwrap().remove(terminal_id); self.terminal_notifications.lock().unwrap().remove(terminal_id); From d7f1b1ff62833ef6f86b9e8074ee40c3be3c7530 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:25:33 -0700 Subject: [PATCH 089/133] fix(tui): serialize terminal retirement with hook admission --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 74d58490df30..4bf395edf82b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10383,10 +10383,11 @@ impl Mux { } fn purge_terminal_side_tables(&self, terminal_id: &TerminalPublicId) -> anyhow::Result<()> { - // Retirement and journal folding share one lifecycle gate. This keeps - // a concurrent fold from installing a candidate snapshot after the - // terminal has been removed. The tombstone is persisted before the - // compatibility cache is cleared. + // Retirement and journal folding share one lifecycle gate. Take the + // hook fence first so teardown follows the same order as ingress: + // `agent_hook_fences` then `agent_roster_fold`. The tombstone is + // persisted before the compatibility cache is cleared. + let mut fences = self.agent_hook_fences.lock().unwrap(); let _fold = self.agent_roster_fold.lock().unwrap(); let (journal_cursor, pending_cleanup) = { let mut registry = self.workspace_registry.lock().unwrap(); @@ -10434,12 +10435,9 @@ impl Mux { } else { None }; - // The fold lock protects the durable tombstone and snapshot only. - // Release it before touching the hook-fence table so teardown never - // takes `agent_roster_fold` then `agent_hook_fences` while ingress - // takes the inverse order. + fences.remove(terminal_id); drop(_fold); - self.agent_hook_fences.lock().unwrap().remove(terminal_id); + drop(fences); self.agent_records.lock().unwrap().remove(terminal_id); self.terminal_notifications.lock().unwrap().remove(terminal_id); snapshot_error.or(pending_cleanup_error).map_or(Ok(()), Err) From 2e95d1b15e07e77985a98c5c919bdc862c9c4c9c Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:04:21 -0700 Subject: [PATCH 090/133] fix(tui): restore external agent lifecycle continuity --- .../crates/cmux-tui-core/src/agent_hooks.rs | 7 +- .../cmux-tui-core/src/journal_reducers.rs | 79 ++++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 297 ++++++++++++------ 3 files changed, 277 insertions(+), 106 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs index b73af2a51235..8c4240d18716 100644 --- a/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs +++ b/cmux-tui/crates/cmux-tui-core/src/agent_hooks.rs @@ -221,8 +221,9 @@ fn validate_agent_source(source: &str) -> anyhow::Result<()> { "agent source must contain 1 to {MAX_AGENT_SOURCE_BYTES} lowercase ASCII letters, digits, hyphens, or underscores" ); anyhow::ensure!( - source != crate::journal_reducers::SOCKET_REPORT_ADAPTER, - "agent source is reserved for internal socket reports" + source != crate::journal_reducers::SOCKET_REPORT_ADAPTER + && source != crate::journal_reducers::DETECTED_REPORT_ADAPTER, + "agent source is reserved for internal report adapters" ); Ok(()) } @@ -888,7 +889,7 @@ mod tests { fn socket_adapter_name_is_reserved_for_internal_reports() { let error = agent_hook_journal_ingress("socket", "PermissionRequest", None, json!({})).unwrap_err(); - assert!(error.to_string().contains("reserved for internal socket reports")); + assert!(error.to_string().contains("reserved for internal report adapters")); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 52b8406e2389..8cd10bff69c0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -45,6 +45,10 @@ pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 6; /// the fold never has to guess a semantic mapping for it. pub(crate) const SOCKET_REPORT_ADAPTER: &str = "socket"; pub(crate) const SOCKET_REPORT_NATIVE_EVENT: &str = "StateReport"; +/// Screen-detection reports use the same durable state-change envelope. Keep +/// a distinct adapter id so the reducer preserves their lower authority. +pub(crate) const DETECTED_REPORT_ADAPTER: &str = "detected"; +pub(crate) const DETECTED_REPORT_NATIVE_EVENT: &str = "StateReport"; fn agent_state_from_str(value: &str) -> Option { Some(match value { @@ -209,17 +213,21 @@ impl AgentRoster { } let Some(terminal_id) = event.terminal_id() else { return Vec::new() }; let socket_echo = event.adapter_id() == Some(SOCKET_REPORT_ADAPTER); + let detected_echo = event.adapter_id() == Some(DETECTED_REPORT_ADAPTER); + let external_echo = socket_echo || detected_echo; if let Some(retired_at) = self.retired_terminals.get(terminal_id).copied() { // A terminal can only become live again through a newer explicit // session start. All other delayed records remain fenced forever. - if socket_echo || event.kind != "agent.session.started" || event.sequence <= retired_at + if external_echo + || event.kind != "agent.session.started" + || event.sequence <= retired_at { return Vec::new(); } self.retired_terminals.remove(terminal_id); self.hook_fences.remove(terminal_id); } - let (state, source, session, agent, updated_at_ms) = if socket_echo { + let (state, source, session, agent, updated_at_ms) = if external_echo { // Socket echo: explicit state and timestamp carried in the // payload, so the roster mirrors the direct projection // commit exactly. The reporter does not know the agent type. @@ -229,13 +237,18 @@ impl AgentRoster { // The socket adapter is authoritative about origin. Do not // trust the user-controlled normalized source field, or it // could bypass hook-over-socket precedence. - let source = AgentSource::Socket; + let source = if socket_echo { AgentSource::Socket } else { AgentSource::Detected }; let updated_at_ms = event .normalized("updated_at_ms") .and_then(|value| value.parse::().ok()) .unwrap_or(event.committed_at_ms); let session = event.normalized("source_session").map(str::to_string); - if self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) { + if self.hook_fences.get(terminal_id).is_some_and(|fence| { + fence.ended + && event + .normalized("source_session") + .is_none_or(|session| session.is_empty() || session == fence.session_id) + }) { return Vec::new(); } (state, source, session, None, updated_at_ms) @@ -258,6 +271,7 @@ impl AgentRoster { { return Vec::new(); } + let public_session = explicit_session.clone(); let session_id = explicit_session .or_else(|| { (!is_session_start) @@ -278,7 +292,7 @@ impl AgentRoster { self.hook_fences.insert( terminal_id.to_string(), RosterFence { - session_id, + session_id: session_id.clone(), sequence: event.sequence, ended: state == AgentState::Done, }, @@ -287,9 +301,9 @@ impl AgentRoster { // receipt from the previous lifecycle must not suppress a later // socket transition after the hook state is gone. self.socket_receipts.remove(terminal_id); - (state, AgentSource::Hook, None, agent, event.committed_at_ms) + (state, AgentSource::Hook, public_session, agent, event.committed_at_ms) }; - if source == AgentSource::Socket + if source != AgentSource::Hook && self .entries .get(terminal_id) @@ -407,6 +421,13 @@ impl AgentRoster { self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) } + pub(crate) fn ended_hook_session(&self, terminal_id: &str) -> Option<&str> { + self.hook_fences + .get(terminal_id) + .filter(|fence| fence.ended) + .map(|fence| fence.session_id.as_str()) + } + /// Return the last accepted socket state for idempotency checks. The /// receipt survives live-entry removal, but only semantic fields are /// retained because report timestamps are intentionally non-semantic. @@ -525,7 +546,25 @@ mod tests { } #[test] - fn ended_hook_fence_survives_snapshot_and_rejects_fresh_socket_echoes() { + fn detected_echo_keeps_detected_authority() { + let subjects = terminal_subject("term_a"); + let payload = json!({ + "adapter": {"id": DETECTED_REPORT_ADAPTER, "version": 1}, + "normalized": { + "state": "working", + "source": "detected", + "source_session": "screen-1" + } + }); + let mut roster = AgentRoster::default(); + let deltas = roster.apply(&hook_event(1, "agent.state.changed", &subjects, &payload)); + assert_eq!(deltas.len(), 1); + assert_eq!(roster.entries["term_a"].agent_source(), AgentSource::Detected); + assert_eq!(roster.entries["term_a"].session.as_deref(), Some("screen-1")); + } + + #[test] + fn ended_hook_fence_rejects_late_socket_but_allows_a_new_session() { let subjects = terminal_subject("term_a"); let old_hook_payload = json!({ "adapter": {"id": "claude", "version": 1}, @@ -548,19 +587,31 @@ mod tests { roster.apply(&hook_event(1, "agent.session.started", &subjects, &old_hook_payload)); roster.apply(&hook_event(5, "agent.session.ended", &subjects, &old_hook_payload)); assert!(roster.has_ended_hook_fence("term_a")); + let deltas = + roster.apply(&hook_event(6, "agent.state.changed", &subjects, &socket_payload)); + assert_eq!(deltas.len(), 1); + assert_eq!(roster.entries["term_a"].session.as_deref(), Some("fresh-socket-session")); + assert!(roster.has_ended_hook_fence("term_a")); + let mut late_payload = socket_payload.clone(); + late_payload["normalized"]["source_session"] = json!("old-hook-session"); assert!( roster - .apply(&hook_event(6, "agent.state.changed", &subjects, &socket_payload)) + .apply(&hook_event(7, "agent.state.changed", &subjects, &late_payload)) .is_empty() ); - - let mut restored = AgentRoster::restore(&roster.snapshot().to_string()).unwrap(); - assert!(restored.has_ended_hook_fence("term_a")); + late_payload["normalized"]["source_session"] = Value::Null; assert!( - restored - .apply(&hook_event(7, "agent.state.changed", &subjects, &socket_payload)) + roster + .apply(&hook_event(8, "agent.state.changed", &subjects, &late_payload)) .is_empty() ); + + let mut restored = AgentRoster::restore(&roster.snapshot().to_string()).unwrap(); + assert!(restored.has_ended_hook_fence("term_a")); + let deltas = + restored.apply(&hook_event(7, "agent.state.changed", &subjects, &socket_payload)); + assert!(deltas.is_empty(), "identical external state is coalesced"); + assert_eq!(restored.entries["term_a"].session.as_deref(), Some("fresh-socket-session")); assert!( restored .apply(&hook_event(5, "agent.session.started", &subjects, &new_hook_payload)) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4bf395edf82b..384affe9bab3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1212,6 +1212,10 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result record.updated_at_ms - }); - if !keep_newer_direct_record { - agent_records.insert(terminal_id, record); - } + // A bounded startup replay may contain an older prefix of the + // journal. Do not let that partial reducer state replace the current + // public projection, which is already restored from the latest + // committed resource mutation. The worker reconciles this cache after + // it reaches the journal head. + if !agent_roster.startup_replay_pending { + merge_agent_roster_into_record_cache(&agent_roster.roster, &mut agent_records); } let journal_producers = registry.journal_producer_manifests()?; let session_public_id = registry.session_id().clone(); @@ -6140,6 +6118,26 @@ impl Mux { continue; } + // The startup cache intentionally stayed on the + // resource projection while the reducer consumed its + // bounded prefix. Reconcile only after this race-safe + // head check proves that the complete journal is folded. + if mux.journal_event_epoch() != observed_epoch { + continue; + } + let startup_replay_completed = { + let mut host = mux.agent_roster.lock().unwrap(); + if host.startup_replay_pending && host.cursor >= current_head { + host.startup_replay_pending = false; + true + } else { + false + } + }; + if startup_replay_completed { + mux.reconcile_agent_record_cache_from_roster(); + } + mux.agent_roster_fold_worker_running.store(false, Ordering::Release); if mux.journal_event_epoch() != observed_epoch && mux @@ -6189,6 +6187,48 @@ impl Mux { Ok(()) } + fn merge_agent_roster_into_record_cache( + roster: &crate::journal_reducers::AgentRoster, + records: &mut HashMap, + ) { + // A durable removal or ended hook fence is stronger than a stale + // compatibility projection. Keep records for terminals with no + // roster decision yet, because direct reports may legitimately be + // newer than the last journal echo. + records.retain(|terminal_id, _| !roster.has_terminal_removal_fence(terminal_id.as_str())); + for (terminal_id, entry) in &roster.entries { + let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) else { continue }; + let record = TerminalAgentRecord { + state: entry.agent_state(), + source: entry.agent_source(), + session: entry.session.clone(), + agent: entry.agent.clone(), + updated_at_ms: entry.updated_at_ms, + }; + // A repeated direct report refreshes the compatibility projection + // without appending a journal echo. Preserve that newer value when + // the roster snapshot still has an older equivalent transition. + // Source equality is required because authority determines which + // projection is allowed to win. + let keep_newer_direct_record = records.get(&terminal_id).is_some_and(|existing| { + matches!(existing.source, AgentSource::Socket | AgentSource::Detected) + && existing.source == record.source + && existing.state == record.state + && existing.session == record.session + && existing.updated_at_ms > record.updated_at_ms + }); + if !keep_newer_direct_record { + records.insert(terminal_id, record); + } + } + } + + fn reconcile_agent_record_cache_from_roster(&self) { + let roster = self.agent_roster.lock().unwrap().roster.clone(); + let mut records = self.agent_records.lock().unwrap(); + Self::merge_agent_roster_into_record_cache(&roster, &mut records); + } + fn apply_roster_delta_to_record_cache(&self, delta: crate::journal_reducers::RosterDelta) { use crate::journal_reducers::RosterDelta; let mut records = self.agent_records.lock().unwrap(); @@ -6264,11 +6304,11 @@ impl Mux { && previous_session.as_deref() == session }, ); - let hook_owned_socket_report = + let hook_owned_external_report = previous.as_ref().is_some_and(|(_, previous_source, _, _)| { - previous_source == AgentSource::Hook.as_str() && source == AgentSource::Socket + previous_source == AgentSource::Hook.as_str() && source != AgentSource::Hook }); - if unchanged || hook_owned_socket_report { + if unchanged || hook_owned_external_report { return None; } let previous = previous @@ -6306,7 +6346,15 @@ impl Mux { updated_at_ms: u64, _idempotency_key: &str, ) -> anyhow::Result { - use crate::journal_reducers::{SOCKET_REPORT_ADAPTER, SOCKET_REPORT_NATIVE_EVENT}; + use crate::journal_reducers::{ + DETECTED_REPORT_ADAPTER, DETECTED_REPORT_NATIVE_EVENT, SOCKET_REPORT_ADAPTER, + SOCKET_REPORT_NATIVE_EVENT, + }; + let (adapter_id, native_event) = match source { + AgentSource::Socket => (SOCKET_REPORT_ADAPTER, SOCKET_REPORT_NATIVE_EVENT), + AgentSource::Detected => (DETECTED_REPORT_ADAPTER, DETECTED_REPORT_NATIVE_EVENT), + AgentSource::Hook => return Err(anyhow::anyhow!("hook reports do not use echoes")), + }; let ingress = crate::JournalIngress { producer_id: crate::agent_hooks::AGENT_HOOK_PRODUCER_ID.into(), manifest_version: crate::agent_hooks::AGENT_HOOK_MANIFEST_VERSION, @@ -6320,8 +6368,8 @@ impl Mux { sensitivity: Some(crate::JournalSensitivity::Sensitive), payload: serde_json::json!({ "format": crate::agent_hooks::AGENT_HOOK_FORMAT, - "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, - "native_event": SOCKET_REPORT_NATIVE_EVENT, + "adapter": {"id": adapter_id, "version": 1}, + "native_event": native_event, "normalized": { "state": state.as_str(), "source": source.as_str(), @@ -9881,14 +9929,14 @@ impl Mux { "source":source.as_str(), "source_session":session, }); - // Socket echo admission and its resource commit must share the same - // fence. Otherwise two equal polls can both derive the same key - // before either one publishes the receipt, and the replay path would - // incorrectly report a missing fresh record. - let socket_sequence_guard = (source == AgentSource::Socket && !sequence_lock_held) + // External echo admission and its resource commit must share the + // same fence. Otherwise two equal reports can both derive the same + // key before either one publishes the receipt, and the replay path + // can incorrectly report a missing fresh record. + let external_sequence_guard = (source != AgentSource::Hook && !sequence_lock_held) .then(|| self.agent_hook_fences.lock().unwrap()); - let commit_sequence_lock_held = sequence_lock_held || socket_sequence_guard.is_some(); - let (echo_sequence, echo_timestamp, echo_ingress) = if source == AgentSource::Socket { + let commit_sequence_lock_held = sequence_lock_held || external_sequence_guard.is_some(); + let (echo_sequence, echo_timestamp, echo_ingress) = if source != AgentSource::Hook { let terminal_id = { let state_guard = self.state.lock().unwrap(); let runtime = state_guard @@ -9937,7 +9985,7 @@ impl Mux { echo_timestamp, echo_ingress.as_ref(), ); - drop(socket_sequence_guard); + drop(external_sequence_guard); let (_, record, fold_receipt) = commit_result?; if let Some(receipt) = fold_receipt { self.fold_agent_roster( @@ -9978,11 +10026,11 @@ impl Mux { "source_session":source_session, }); // Hold the hook fence while deriving the semantic echo key and - // committing it. This closes the equal-poll race without adding a + // committing it. This closes the equal-report race without adding a // second, independent serialization primitive. - let socket_sequence_guard = - (source == AgentSource::Socket).then(|| self.agent_hook_fences.lock().unwrap()); - let (journal_sequence, reported_at_ms, echo_ingress) = if source == AgentSource::Socket { + let external_sequence_guard = + (source != AgentSource::Hook).then(|| self.agent_hook_fences.lock().unwrap()); + let (journal_sequence, reported_at_ms, echo_ingress) = if source != AgentSource::Hook { let timestamp = now_ms(); match self.agent_report_echo_key( terminal_id, @@ -10014,7 +10062,7 @@ impl Mux { expected_revision, mutation, &fingerprint, - socket_sequence_guard.is_some(), + external_sequence_guard.is_some(), None, journal_sequence, AgentReportOrigin::Direct, @@ -10022,7 +10070,7 @@ impl Mux { reported_at_ms, echo_ingress.as_ref(), ); - drop(socket_sequence_guard); + drop(external_sequence_guard); if result.is_ok() && source != AgentSource::Hook { let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); } @@ -10123,7 +10171,9 @@ impl Mux { if roster.roster.is_retired(terminal_id.as_str()) { anyhow::bail!("terminal {terminal_id} has been retired"); } - let durable_hook_end = roster.roster.has_ended_hook_fence(terminal_id.as_str()); + let durable_hook_session = + roster.roster.ended_hook_session(terminal_id.as_str()).map(str::to_owned); + let durable_hook_end = durable_hook_session.is_some(); drop(roster); let mut direct_hook_state = None; if source != AgentSource::Hook { @@ -10131,7 +10181,14 @@ impl Mux { .as_ref() .and_then(|guard| guard.get(&terminal_id)) .filter(|fence| fence.ended); - if durable_hook_end || ended_fence.is_some() { + let new_lifecycle_session = source_session.as_deref().is_some_and(|session| { + !session.is_empty() + && !session.starts_with("cmux-hook-sequence:") + && !session.starts_with("cmux-hook-ended:") + && !ended_fence.is_some_and(|fence| session == fence.session_id) + && !durable_hook_session.as_deref().is_some_and(|old| session == old) + }); + if (durable_hook_end || ended_fence.is_some()) && !new_lifecycle_session { anyhow::bail!("agent_session_ended"); } } else if let Some(fence) = sequence_guard @@ -10168,18 +10225,36 @@ impl Mux { } } let effective_hook_state = direct_hook_state.as_ref().or(hook_state); - let persisted_source_session = if source == AgentSource::Hook { - source_session.clone().filter(|value| { - !value.starts_with("cmux-hook-sequence:") && !value.starts_with("cmux-hook-ended:") - }) + let source_session = if source == AgentSource::Hook { + // Internal sequence markers are transport metadata. Expose the + // validated native identity from hook state to clients, while + // keeping generated legacy identities private. + source_session + .filter(|value| { + !value.starts_with("cmux-hook-sequence:") + && !value.starts_with("cmux-hook-ended:") + }) + .or_else(|| { + hook_state.and_then(|state| { + (!state.agent_session_id.starts_with("legacy:")) + .then(|| state.agent_session_id.clone()) + }) + }) } else { if source_session.as_deref().is_some_and(|value| { value.starts_with("cmux-hook-sequence:") || value.starts_with("cmux-hook-ended:") }) { anyhow::bail!("reserved hook marker is invalid for non-hook agent source"); } - // Preserve a valid fresh socket identity. The internal hook - // marker is only a compatibility fallback when no identity was + source_session.filter(|value| { + !value.starts_with("cmux-hook-sequence:") && !value.starts_with("cmux-hook-ended:") + }) + }; + let persisted_source_session = if source == AgentSource::Hook { + source_session.clone() + } else { + // Preserve a valid external identity. The internal hook marker + // is only a compatibility fallback when no identity was // supplied, while durable hook state carries the fence itself. source_session.clone().or_else(|| { sequence_guard @@ -10188,9 +10263,6 @@ impl Mux { .map(|fence| format!("cmux-hook-sequence:{}", fence.sequence)) }) }; - let source_session = source_session.filter(|value| { - !value.starts_with("cmux-hook-sequence:") && !value.starts_with("cmux-hook-ended:") - }); let now = reported_at_ms.unwrap_or_else(now_ms); let mut records = self.agent_records.lock().unwrap(); let lower_authority_report_ignored = records.get(&terminal_id).is_some_and(|existing| { @@ -23051,7 +23123,7 @@ mod tests { let filtered = mux.list_agents(Some(surface.id), Some(AgentState::Blocked)); assert_eq!(filtered.len(), 1); - assert_eq!(filtered[0].session, None); + assert_eq!(filtered[0].session.as_deref(), Some("hook-session")); assert!(mux.list_agents(Some(surface.id), Some(AgentState::Done)).is_empty()); assert_eq!(mux.with_state(|state| state.resource_revision), initial_revision + 3); assert_eq!(mux.resource_event_epoch(), initial_epoch + 3); @@ -23062,14 +23134,17 @@ mod tests { assert_eq!(resource_events.batches[1].changes[0]["value"]["source"], "hook"); assert_eq!(resource_events.batches[2].changes[0]["value"]["source"], "hook"); assert_eq!(resource_events.batches[2].changes[0]["value"]["state"], "blocked"); - assert_eq!(resource_events.batches[2].changes[0]["value"]["source_session"], Value::Null); + assert_eq!( + resource_events.batches[2].changes[0]["value"]["source_session"], + "hook-session" + ); assert!(matches!( events.recv_timeout(Duration::from_millis(100)), Ok(MuxEvent::AgentChanged { surface: event_surface, state, source, - session: None, + session: Some(session), .. }) if event_surface == surface.id && state.as_ref() == "blocked" @@ -23265,7 +23340,7 @@ mod tests { assert_eq!(records.len(), 1); assert_eq!(records[0].state, AgentState::Blocked); assert_eq!(records[0].source, AgentSource::Hook); - assert_eq!(records[0].session, None); + assert_eq!(records[0].session.as_deref(), Some("racing-hook")); assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 1); let batches = mux.resource_events_after(revision).unwrap().batches; assert_eq!(batches.len(), 2); @@ -24895,7 +24970,7 @@ mod tests { } #[test] - fn fresh_socket_session_remains_fenced_until_explicit_hook_start() { + fn fresh_external_session_can_follow_hook_end_and_survives_restart() { let root = std::env::temp_dir() .join(format!("cmux-agent-hook-restart-{}", crate::workspace_registry::new_uuid_v4())); let mux = @@ -24913,22 +24988,29 @@ mod tests { }; mux.append_journal_ingress(&hook("SessionEnd", "old-hook"), "test", "ended-hook").unwrap(); - assert!( - mux.report_agent( + let record = mux + .report_agent( surface.id, AgentState::Working, AgentSource::Socket, Some("new-socket-session".into()), ) - .is_err() - ); - assert!(mux.list_agents(Some(surface.id), None).is_empty()); + .unwrap(); + assert_eq!(record.source, AgentSource::Socket); + assert_eq!(record.session.as_deref(), Some("new-socket-session")); mux.shutdown(); drop(mux); let reopened = Mux::open_persistent("agent-hook-restart", SurfaceOptions::default(), &root).unwrap(); let reopened_surface = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); + let reopened_record = reopened + .list_agents(Some(reopened_surface), None) + .into_iter() + .next() + .expect("socket transition is restored from its journal echo"); + assert_eq!(reopened_record.source, AgentSource::Socket); + assert_eq!(reopened_record.session.as_deref(), Some("new-socket-session")); assert!( reopened .report_agent( @@ -24937,9 +25019,8 @@ mod tests { AgentSource::Socket, Some("new-socket-session".into()), ) - .is_err() + .is_ok() ); - assert!(reopened.list_agents(Some(reopened_surface), None).is_empty()); reopened .append_journal_ingress(&hook("SessionStart", "new-hook"), "test", "new-hook-start") @@ -24952,19 +25033,57 @@ mod tests { .roster .has_ended_hook_fence(terminal_id.as_str()) ); - assert!( - reopened - .report_agent( - reopened_surface, - AgentState::Working, - AgentSource::Socket, - Some("new-socket-session".into()), - ) - .is_ok() - ); - let reopened_record = &reopened.list_agents(Some(reopened_surface), None)[0]; - assert_eq!(reopened_record.source, AgentSource::Hook); + let hook_record = reopened + .list_agents(Some(reopened_surface), None) + .into_iter() + .next() + .expect("new hook session is live"); + assert_eq!(hook_record.source, AgentSource::Hook); + reopened.shutdown(); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn detected_session_can_follow_hook_end_and_survives_restart() { + let root = std::env::temp_dir().join(format!( + "cmux-agent-detected-restart-{}", + crate::workspace_registry::new_uuid_v4() + )); + let mux = Mux::open_persistent("agent-detected-restart", SurfaceOptions::default(), &root) + .unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + let ended = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "SessionEnd", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"old-hook"}), + ) + .unwrap(); + mux.append_journal_ingress(&ended, "test", "detected-ended").unwrap(); + let record = mux + .report_agent( + surface.id, + AgentState::Working, + AgentSource::Detected, + Some("new-screen-session".into()), + ) + .unwrap(); + assert_eq!(record.source, AgentSource::Detected); + assert_eq!(record.session.as_deref(), Some("new-screen-session")); + mux.shutdown(); + drop(mux); + + let reopened = + Mux::open_persistent("agent-detected-restart", SurfaceOptions::default(), &root) + .unwrap(); + let reopened_surface = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); + let restored = reopened.list_agents(Some(reopened_surface), None); + assert_eq!(restored.len(), 1); + assert_eq!(restored[0].source, AgentSource::Detected); + assert_eq!(restored[0].session.as_deref(), Some("new-screen-session")); reopened.shutdown(); + drop(reopened); std::fs::remove_dir_all(root).unwrap(); } @@ -24992,7 +25111,7 @@ mod tests { .unwrap(); let record = &mux.list_agents(Some(surface.id), None)[0]; assert_eq!(record.source, AgentSource::Hook); - assert_eq!(record.session, None); + assert_eq!(record.session.as_deref(), Some("new")); mux.append_journal_ingress(&hook("UserPromptSubmit", "new"), "test", "journal-hook-turn") .unwrap(); assert_eq!(mux.agent_hook_fences.lock().unwrap()[&terminal_id].session_id, "new"); From f496d666bce9a5febe13a3fb93827e286da76d42 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:30:52 -0700 Subject: [PATCH 091/133] fix: qualify roster cache helper --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 384affe9bab3..44e9c5e11ad3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2575,7 +2575,7 @@ impl Mux { // committed resource mutation. The worker reconciles this cache after // it reaches the journal head. if !agent_roster.startup_replay_pending { - merge_agent_roster_into_record_cache(&agent_roster.roster, &mut agent_records); + Self::merge_agent_roster_into_record_cache(&agent_roster.roster, &mut agent_records); } let journal_producers = registry.journal_producer_manifests()?; let session_public_id = registry.session_id().clone(); From 138fffca97e38b096ef56770d3627c35b6832f2d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:19:45 -0700 Subject: [PATCH 092/133] fix(cmux-tui): serialize roster fences through replay --- .../cmux-tui-core/src/journal_reducers.rs | 139 ++++++++-- cmux-tui/crates/cmux-tui-core/src/mux.rs | 251 +++++++++++++----- 2 files changed, 301 insertions(+), 89 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 8cd10bff69c0..4fbd46c2417f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -32,7 +32,9 @@ pub(crate) const AGENT_ROSTER_REDUCER_ID: &str = "agent_roster"; /// added a global authority bit. Version 5 removes that global bit and scopes /// compatibility cleanup to each terminal's durable removal fence. Version 6 /// retains the last socket semantic receipt after a Done entry is removed. -pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 6; +/// Version 7 makes that receipt source-neutral for socket and detection +/// adapters. +pub(crate) const AGENT_ROSTER_REDUCER_VERSION: u32 = 7; /// Retirement tombstones protect delayed journal rows after a terminal leaves /// the resource tree. Keep each exact tombstone while its journal rows may be @@ -138,7 +140,8 @@ struct RosterFence { } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -struct SocketReportReceipt { +struct ExternalReportReceipt { + source: String, state: String, session: Option, } @@ -195,12 +198,13 @@ pub(crate) struct AgentRoster { /// retirement, so delayed records from that terminal cannot recreate it. #[serde(default)] retired_terminals: BTreeMap, - /// The last accepted socket state remains after a Done report removes + /// The last accepted external state remains after a Done report removes /// the live roster entry. This receipt is bounded by terminal lifecycle, /// and lets repeated polls reuse the same semantic transition instead of - /// appending another journal row on every poll. + /// appending another journal row on every poll. It covers socket and + /// screen-detection adapters. #[serde(default)] - socket_receipts: HashMap, + external_receipts: HashMap, } impl AgentRoster { @@ -243,6 +247,26 @@ impl AgentRoster { .and_then(|value| value.parse::().ok()) .unwrap_or(event.committed_at_ms); let session = event.normalized("source_session").map(str::to_string); + if self + .external_receipts + .get(terminal_id) + .is_some_and(|receipt| receipt.state == AgentState::Done.as_str()) + { + // An external Done closes that source session. A later + // report must prove a distinct non-empty session before it + // can start a new lifecycle on the same terminal. + let fresh_session = session.as_deref().is_some_and(|session| { + !session.is_empty() + && self + .external_receipts + .get(terminal_id) + .and_then(|receipt| receipt.session.as_deref()) + != Some(session) + }); + if !fresh_session { + return Vec::new(); + } + } if self.hook_fences.get(terminal_id).is_some_and(|fence| { fence.ended && event @@ -300,7 +324,7 @@ impl AgentRoster { // A hook event claims this terminal's lifecycle. Any socket // receipt from the previous lifecycle must not suppress a later // socket transition after the hook state is gone. - self.socket_receipts.remove(terminal_id); + self.external_receipts.remove(terminal_id); (state, AgentSource::Hook, public_session, agent, event.committed_at_ms) }; if source != AgentSource::Hook @@ -313,17 +337,33 @@ impl AgentRoster { // overwrite it (mirrors the projection commit precedence). return Vec::new(); } - if source == AgentSource::Socket { - self.socket_receipts.insert( - terminal_id.to_string(), - SocketReportReceipt { state: state.as_str().to_string(), session: session.clone() }, - ); + let external_receipt_changed = if source != AgentSource::Hook { + let receipt = ExternalReportReceipt { + source: source.as_str().to_string(), + state: state.as_str().to_string(), + session: session.clone(), + }; + let changed = self.external_receipts.get(terminal_id) != Some(&receipt); + if changed { + self.external_receipts.insert(terminal_id.to_string(), receipt); + } + changed + } else { + false + }; + if state == AgentState::Done && source != AgentSource::Hook && !external_receipt_changed { + // A repeated external Done is already represented by its durable + // receipt. Do not emit another removal delta. + return Vec::new(); } if state == AgentState::Done { // An ended agent leaves the roster entirely; the done state is // still committed to the durable projection by the host so // history and remote caches converge. - return if self.entries.remove(terminal_id).is_some() { + return if self.entries.remove(terminal_id).is_some() + || source == AgentSource::Hook + || external_receipt_changed + { vec![RosterDelta::Remove { terminal_id: terminal_id.to_string() }] } else { Vec::new() @@ -361,7 +401,7 @@ impl AgentRoster { /// events yet, so the host retires entries explicitly. pub(crate) fn retire_terminal(&mut self, terminal_id: &str, retired_at: u64) -> bool { let removed_entry = self.entries.remove(terminal_id).is_some(); - let removed_socket_receipt = self.socket_receipts.remove(terminal_id).is_some(); + let removed_external_receipt = self.external_receipts.remove(terminal_id).is_some(); // A retired terminal is no longer allowed to receive socket echoes, // and its ended hook fence is redundant with the retirement cursor. let removed_fence = self.hook_fences.remove(terminal_id).is_some(); @@ -377,7 +417,7 @@ impl AgentRoster { true } }; - removed_entry || removed_fence || removed_socket_receipt || retired_changed + removed_entry || removed_fence || removed_external_receipt || retired_changed } /// Remove retirement fences whose source records are covered by a sealed @@ -399,6 +439,10 @@ impl AgentRoster { pub(crate) fn has_terminal_removal_fence(&self, terminal_id: &str) -> bool { self.retired_terminals.contains_key(terminal_id) || self.hook_fences.get(terminal_id).is_some_and(|fence| fence.ended) + || self + .external_receipts + .get(terminal_id) + .is_some_and(|receipt| receipt.state == AgentState::Done.as_str()) } /// Return every terminal represented by durable roster state. Startup @@ -407,11 +451,15 @@ impl AgentRoster { /// being rebuilt. pub(crate) fn fenced_terminal_ids(&self) -> Vec { let mut ids = Vec::with_capacity( - self.entries.len() + self.hook_fences.len() + self.retired_terminals.len(), + self.entries.len() + + self.hook_fences.len() + + self.retired_terminals.len() + + self.external_receipts.len(), ); ids.extend(self.entries.keys().cloned()); ids.extend(self.hook_fences.keys().cloned()); ids.extend(self.retired_terminals.keys().cloned()); + ids.extend(self.external_receipts.keys().cloned()); ids.sort_unstable(); ids.dedup(); ids @@ -428,13 +476,23 @@ impl AgentRoster { .map(|fence| fence.session_id.as_str()) } - /// Return the last accepted socket state for idempotency checks. The + /// Return the last accepted external state for idempotency checks. The /// receipt survives live-entry removal, but only semantic fields are /// retained because report timestamps are intentionally non-semantic. - pub(crate) fn last_socket_report(&self, terminal_id: &str) -> Option<(&str, Option<&str>)> { - self.socket_receipts + pub(crate) fn last_external_report( + &self, + terminal_id: &str, + ) -> Option<(&str, &str, Option<&str>)> { + self.external_receipts.get(terminal_id).map(|receipt| { + (receipt.source.as_str(), receipt.state.as_str(), receipt.session.as_deref()) + }) + } + + pub(crate) fn ended_external_session(&self, terminal_id: &str) -> Option<&str> { + self.external_receipts .get(terminal_id) - .map(|receipt| (receipt.state.as_str(), receipt.session.as_deref())) + .filter(|receipt| receipt.state == AgentState::Done.as_str()) + .and_then(|receipt| receipt.session.as_deref()) } pub(crate) fn snapshot(&self) -> Value { @@ -563,6 +621,49 @@ mod tests { assert_eq!(roster.entries["term_a"].session.as_deref(), Some("screen-1")); } + #[test] + fn external_done_fence_survives_restart_and_requires_a_new_session() { + let subjects = terminal_subject("term_a"); + let done = json!({ + "adapter": {"id": DETECTED_REPORT_ADAPTER, "version": 1}, + "normalized": { + "state": "done", + "source": "detected", + "source_session": "screen-1" + } + }); + let mut roster = AgentRoster::default(); + + assert_eq!( + roster.apply(&hook_event(1, "agent.state.changed", &subjects, &done)), + vec![RosterDelta::Remove { terminal_id: "term_a".into() }] + ); + assert!(roster.entries.is_empty()); + assert!(roster.has_terminal_removal_fence("term_a")); + + let mut restored = AgentRoster::restore(&roster.snapshot().to_string()).unwrap(); + assert!(restored.apply(&hook_event(2, "agent.state.changed", &subjects, &done)).is_empty()); + + let mut stale_working = done.clone(); + stale_working["normalized"]["state"] = json!("working"); + assert!( + restored + .apply(&hook_event(3, "agent.state.changed", &subjects, &stale_working)) + .is_empty() + ); + + let mut fresh_working = stale_working; + fresh_working["normalized"]["source_session"] = json!("screen-2"); + assert_eq!( + restored.apply(&hook_event(4, "agent.state.changed", &subjects, &fresh_working)), + vec![RosterDelta::Upsert { + terminal_id: "term_a".into(), + entry: restored.entries["term_a"].clone(), + }] + ); + assert_eq!(restored.entries["term_a"].session.as_deref(), Some("screen-2")); + } + #[test] fn ended_hook_fence_rejects_late_socket_but_allows_a_new_session() { let subjects = terminal_subject("term_a"); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 44e9c5e11ad3..519675ea545f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1351,10 +1351,9 @@ enum AgentReportOrigin { RosterFold, } -/// The portion of an agent report that must be folded after all report -/// serialization locks are released. Keeping this receipt separate from the -/// resource transaction prevents a socket fence from being held while the -/// journal reducer takes its own lock. +/// The portion of an agent report that must be folded after the resource +/// transaction commits. Keeping this receipt separate lets the reducer use +/// its bounded journal path while the external report fence remains held. struct AgentReportFoldReceipt { ingress: crate::JournalIngress, sequence: u64, @@ -2269,6 +2268,9 @@ pub struct Mux { /// A single coalesced worker drains replay work that exceeds the bounded /// synchronous report budget. The journal remains the source of truth. agent_roster_fold_worker_running: AtomicBool, + /// Signals shutdown waiters after the coalesced roster worker releases + /// its ownership flag. The fold mutex is the associated wait lock. + agent_roster_fold_worker_changed: Condvar, /// Weak self-reference used to start the reducer worker from `&self` /// entrypoints without creating a reference cycle. It is initialized once /// immediately after the owning `Arc` is constructed. @@ -2685,6 +2687,7 @@ impl Mux { agent_roster: Mutex::new(agent_roster), agent_roster_fold: Mutex::new(()), agent_roster_fold_worker_running: AtomicBool::new(false), + agent_roster_fold_worker_changed: Condvar::new(), self_reference: OnceLock::new(), agent_records: Mutex::new(agent_records), agent_hook_fences: Mutex::new(agent_hook_fences), @@ -5886,7 +5889,10 @@ impl Mux { // complete roster for every poll made callback cost grow with the // number of terminals. let _fold = self.agent_roster_fold.lock().unwrap(); - let mut read_cursor = self.agent_roster.lock().unwrap().cursor; + let (mut read_cursor, defer_record_cache) = { + let host = self.agent_roster.lock().unwrap(); + (host.cursor, host.startup_replay_pending) + }; if target_sequence <= read_cursor { return AgentRosterFoldProgress::ReachedTarget; } @@ -5972,8 +5978,10 @@ impl Mux { matches!(delta, crate::journal_reducers::RosterDelta::Remove { .. }) }); } - for delta in deltas { - self.apply_roster_delta_to_record_cache(delta); + if !defer_record_cache { + for delta in deltas { + self.apply_roster_delta_to_record_cache(delta); + } } read_cursor = record.sequence; advanced = true; @@ -6041,23 +6049,51 @@ impl Mux { } let Some(mux) = self.self_reference.get().cloned() else { - self.agent_roster_fold_worker_running.store(false, Ordering::Release); + self.mark_agent_roster_fold_worker_stopped(); return; }; if let Err(error) = std::thread::Builder::new() .name("agent-roster-fold".into()) .spawn(move || Self::run_agent_roster_fold_worker(mux)) { - self.agent_roster_fold_worker_running.store(false, Ordering::Release); + self.mark_agent_roster_fold_worker_stopped(); eprintln!("cmux-tui: could not start agent roster fold worker: {error}"); } } + fn mark_agent_roster_fold_worker_stopped(&self) { + let _fold = self.agent_roster_fold.lock().unwrap(); + self.agent_roster_fold_worker_running.store(false, Ordering::Release); + self.agent_roster_fold_worker_changed.notify_all(); + } + + fn wait_for_agent_roster_fold_worker(&self, deadline: Instant) { + let mut fold = self.agent_roster_fold.lock().unwrap(); + while self.agent_roster_fold_worker_running.load(Ordering::Acquire) { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + eprintln!( + "cmux-tui: agent roster fold worker did not stop before shutdown deadline" + ); + break; + } + let (next, result) = + self.agent_roster_fold_worker_changed.wait_timeout(fold, remaining).unwrap(); + fold = next; + if result.timed_out() && self.agent_roster_fold_worker_running.load(Ordering::Acquire) { + eprintln!( + "cmux-tui: agent roster fold worker did not stop before shutdown deadline" + ); + break; + } + } + } + fn run_agent_roster_fold_worker(mux: Weak) { loop { let Some(mux) = mux.upgrade() else { return }; if mux.shutting_down.load(Ordering::Acquire) { - mux.agent_roster_fold_worker_running.store(false, Ordering::Release); + mux.mark_agent_roster_fold_worker_stopped(); return; } @@ -6125,21 +6161,41 @@ impl Mux { if mux.journal_event_epoch() != observed_epoch { continue; } - let startup_replay_completed = { - let mut host = mux.agent_roster.lock().unwrap(); - if host.startup_replay_pending && host.cursor >= current_head { - host.startup_replay_pending = false; - true - } else { - false - } - }; - if startup_replay_completed { + let startup_replay_ready = + mux.agent_roster.lock().unwrap().startup_replay_pending; + if startup_replay_ready { + // Keep the pending marker set while reconciling. A + // concurrent direct report can update the cache before + // its journal echo is folded; the second epoch check + // then makes this pass retry without exposing a + // partially replayed cache. mux.reconcile_agent_record_cache_from_roster(); + if mux.journal_event_epoch() != observed_epoch { + continue; + } + let current_head = match mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + { + Ok(page) => page.head_sequence, + Err(error) => { + eprintln!( + "cmux-tui: rechecking the agent roster journal head after cache reconciliation failed: {error}" + ); + continue; + } + }; + if mux.agent_roster.lock().unwrap().cursor < current_head { + continue; + } + mux.agent_roster.lock().unwrap().startup_replay_pending = false; } - mux.agent_roster_fold_worker_running.store(false, Ordering::Release); - if mux.journal_event_epoch() != observed_epoch + mux.mark_agent_roster_fold_worker_stopped(); + if !mux.shutting_down.load(Ordering::Acquire) + && mux.journal_event_epoch() != observed_epoch && mux .agent_roster_fold_worker_running .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -6286,14 +6342,16 @@ impl Mux { ) }) .or_else(|| { - host.roster.last_socket_report(terminal_id.as_str()).map(|(state, session)| { - ( - state.to_string(), - AgentSource::Socket.as_str().to_string(), - session.map(str::to_owned), - None, - ) - }) + host.roster.last_external_report(terminal_id.as_str()).map( + |(source, state, session)| { + ( + state.to_string(), + source.to_string(), + session.map(str::to_owned), + None, + ) + }, + ) }); previous }; @@ -6512,6 +6570,7 @@ impl Mux { AgentSource::Hook, Some(marker), true, + Some(&*fences), Some(hook_state), Some(sequence), AgentReportOrigin::RosterFold, @@ -9901,6 +9960,7 @@ impl Mux { false, None, None, + None, AgentReportOrigin::Direct, None, ) @@ -9913,6 +9973,7 @@ impl Mux { source: AgentSource, session: Option, sequence_lock_held: bool, + held_fences: Option<&HashMap>, hook_state: Option, journal_sequence: Option, origin: AgentReportOrigin, @@ -9936,6 +9997,8 @@ impl Mux { let external_sequence_guard = (source != AgentSource::Hook && !sequence_lock_held) .then(|| self.agent_hook_fences.lock().unwrap()); let commit_sequence_lock_held = sequence_lock_held || external_sequence_guard.is_some(); + let held_fences = + held_fences.or_else(|| external_sequence_guard.as_ref().map(|guard| &**guard)); let (echo_sequence, echo_timestamp, echo_ingress) = if source != AgentSource::Hook { let terminal_id = { let state_guard = self.state.lock().unwrap(); @@ -9978,6 +10041,7 @@ impl Mux { &mutation, &fingerprint, commit_sequence_lock_held, + held_fences, hook_state.as_ref(), echo_sequence, origin, @@ -9985,7 +10049,6 @@ impl Mux { echo_timestamp, echo_ingress.as_ref(), ); - drop(external_sequence_guard); let (_, record, fold_receipt) = commit_result?; if let Some(receipt) = fold_receipt { self.fold_agent_roster( @@ -9997,6 +10060,7 @@ impl Mux { }, ); } + drop(external_sequence_guard); let record = record.context("fresh raw agent report unexpectedly replayed")?; if source != AgentSource::Hook { // A successful report means this terminal is available. Retry only @@ -10063,6 +10127,7 @@ impl Mux { mutation, &fingerprint, external_sequence_guard.is_some(), + external_sequence_guard.as_ref().map(|guard| &**guard), None, journal_sequence, AgentReportOrigin::Direct, @@ -10070,10 +10135,6 @@ impl Mux { reported_at_ms, echo_ingress.as_ref(), ); - drop(external_sequence_guard); - if result.is_ok() && source != AgentSource::Hook { - let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); - } match result { Ok((commit, _, fold_receipt)) => { if let Some(receipt) = fold_receipt { @@ -10086,9 +10147,16 @@ impl Mux { }, ); } + drop(external_sequence_guard); + if source != AgentSource::Hook { + let _ = self.retry_pending_agent_hooks_for_terminal(terminal_id); + } Ok(commit) } - Err(error) => Err(error), + Err(error) => { + drop(external_sequence_guard); + Err(error) + } } } @@ -10103,6 +10171,7 @@ impl Mux { mutation: &WorkspaceMutation, fingerprint: &Value, sequence_lock_held: bool, + held_fences: Option<&HashMap>, hook_state: Option<&crate::workspace_registry::AgentHookProjectionState>, journal_sequence: Option, origin: AgentReportOrigin, @@ -10167,35 +10236,50 @@ impl Mux { (surface, (*terminal_id).clone()) } }; - let roster = self.agent_roster.lock().unwrap(); - if roster.roster.is_retired(terminal_id.as_str()) { - anyhow::bail!("terminal {terminal_id} has been retired"); - } - let durable_hook_session = - roster.roster.ended_hook_session(terminal_id.as_str()).map(str::to_owned); - let durable_hook_end = durable_hook_session.is_some(); - drop(roster); + let (durable_hook_session, durable_external_session, durable_external_end) = { + let roster = self.agent_roster.lock().unwrap(); + if roster.roster.is_retired(terminal_id.as_str()) { + anyhow::bail!("terminal {terminal_id} has been retired"); + } + let durable_hook_session = + roster.roster.ended_hook_session(terminal_id.as_str()).map(str::to_owned); + let durable_external_session = + roster.roster.ended_external_session(terminal_id.as_str()).map(str::to_owned); + let durable_external_end = roster + .roster + .last_external_report(terminal_id.as_str()) + .is_some_and(|(_, state, _)| state == AgentState::Done.as_str()); + (durable_hook_session, durable_external_session, durable_external_end) + }; + // A caller-held fence is just as authoritative as one acquired in + // this function. Clone the entry before any later mutable insertion + // so the validation decision cannot race or borrow the guard. + let current_fence = held_fences + .or_else(|| sequence_guard.as_ref().map(|guard| &**guard)) + .and_then(|fences| fences.get(&terminal_id)) + .cloned(); + let ended_fence = current_fence.as_ref().filter(|fence| fence.ended); let mut direct_hook_state = None; if source != AgentSource::Hook { - let ended_fence = sequence_guard - .as_ref() - .and_then(|guard| guard.get(&terminal_id)) - .filter(|fence| fence.ended); let new_lifecycle_session = source_session.as_deref().is_some_and(|session| { !session.is_empty() && !session.starts_with("cmux-hook-sequence:") && !session.starts_with("cmux-hook-ended:") && !ended_fence.is_some_and(|fence| session == fence.session_id) && !durable_hook_session.as_deref().is_some_and(|old| session == old) + && !durable_external_session.as_deref().is_some_and(|old| session == old) }); - if (durable_hook_end || ended_fence.is_some()) && !new_lifecycle_session { + if (durable_hook_session.is_some() || durable_external_end || ended_fence.is_some()) + && !new_lifecycle_session + && !(durable_external_end + && durable_hook_session.is_none() + && ended_fence.is_none() + && agent_state == AgentState::Done + && durable_external_session.as_deref() == source_session.as_deref()) + { anyhow::bail!("agent_session_ended"); } - } else if let Some(fence) = sequence_guard - .as_ref() - .and_then(|guard| guard.get(&terminal_id)) - .filter(|fence| fence.ended) - { + } else if let Some(fence) = ended_fence { let supplied_session = source_session.as_deref().filter(|session| { !session.is_empty() && !session.starts_with("cmux-hook-sequence:") @@ -10257,10 +10341,7 @@ impl Mux { // is only a compatibility fallback when no identity was // supplied, while durable hook state carries the fence itself. source_session.clone().or_else(|| { - sequence_guard - .as_ref() - .and_then(|guard| guard.get(&terminal_id)) - .map(|fence| format!("cmux-hook-sequence:{}", fence.sequence)) + current_fence.as_ref().map(|fence| format!("cmux-hook-sequence:{}", fence.sequence)) }) }; let now = reported_at_ms.unwrap_or_else(now_ms); @@ -10401,9 +10482,8 @@ impl Mux { drop(records); drop(state); drop(registry); - // The fence protects the commit only. Release it before folding the - // journal row so a future fold implementation cannot re-enter the - // report serialization domain. + // Keep the fence through the synchronous fold. The fold publishes the + // durable receipt that the next external report uses for coalescing. drop(sequence_guard); let publish_echo = !commit.replayed && echo_sequence.is_some(); if publish_echo { @@ -10592,6 +10672,7 @@ impl Mux { self.journal_event_changed.notify_all(); self.journal_kernel.wake_waiters(); let hook_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; + self.wait_for_agent_roster_fold_worker(hook_deadline); if !self.journal_hook_runtime.shutdown_until(hook_deadline) { eprintln!("cmux-tui: journal hook workers did not stop before the shutdown deadline"); } @@ -23476,10 +23557,24 @@ mod tests { true, ) .unwrap(); - let surface_id = reopened.resource_surface_for_terminal(&terminal_id).unwrap(); - let records = reopened.list_agents(Some(surface_id), None); - assert_eq!(records.len(), 1); - assert_eq!(records[0].agent.as_deref(), Some("claude")); + // A restarted daemon without a live host record detaches the + // terminal topology. The durable roster and compatibility cache must + // still retain the identity independently of that host state. + let roster = reopened.agent_roster.lock().unwrap(); + assert_eq!( + roster + .roster + .entries + .get(terminal_id.as_str()) + .and_then(|entry| entry.agent.as_deref()), + Some("claude") + ); + drop(roster); + let records = reopened.agent_records.lock().unwrap(); + assert_eq!( + records.get(&terminal_id).and_then(|record| record.agent.as_deref()), + Some("claude") + ); reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); @@ -24270,6 +24365,16 @@ mod tests { .unwrap(); mux.append_journal_ingress(&end, "test", "authority-scope-end").unwrap(); + // Keep a live external projection on a different terminal. Ending + // one terminal must not clear this unrelated compatibility record. + mux.report_agent( + compatibility_surface.id, + AgentState::Working, + AgentSource::Socket, + Some("compatibility-session".into()), + ) + .unwrap(); + // Direct hook reports are rejected because they have no durable // journal event to replay after restart. assert!( @@ -24285,11 +24390,17 @@ mod tests { drop(mux); let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); - assert!(reopened.list_agents(Some(ended_surface.id), None).is_empty()); - let compatibility_surface = reopened - .resource_surface_for_terminal(&compatibility_terminal) - .expect("compatibility terminal restored"); - assert!(reopened.list_agents(Some(compatibility_surface), None).is_empty()); + let records = reopened.agent_records.lock().unwrap(); + assert!(!records.contains_key(&ended_terminal)); + assert_eq!( + records.get(&compatibility_terminal).map(|record| ( + record.state, + record.source, + record.session.as_deref() + )), + Some((AgentState::Working, AgentSource::Socket, Some("compatibility-session"))) + ); + drop(records); reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); From fbc5edcf126509a74de5fe91748332497f6ae99c Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:19:45 -0700 Subject: [PATCH 093/133] fix(cmux-tui): serialize roster fences through replay --- .../cmux-tui-core/src/journal_reducers.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 4fbd46c2417f..ff4d0ea86abd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -654,13 +654,16 @@ mod tests { let mut fresh_working = stale_working; fresh_working["normalized"]["source_session"] = json!("screen-2"); - assert_eq!( - restored.apply(&hook_event(4, "agent.state.changed", &subjects, &fresh_working)), - vec![RosterDelta::Upsert { - terminal_id: "term_a".into(), - entry: restored.entries["term_a"].clone(), - }] - ); + let deltas = + restored.apply(&hook_event(4, "agent.state.changed", &subjects, &fresh_working)); + assert_eq!(deltas.len(), 1); + assert!(matches!( + &deltas[0], + RosterDelta::Upsert { terminal_id, entry } + if terminal_id == "term_a" + && entry.session.as_deref() == Some("screen-2") + && entry.state == "working" + )); assert_eq!(restored.entries["term_a"].session.as_deref(), Some("screen-2")); } From ff9f35591ded404ca4c1f22bab7d36ad3cdcfca7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:27:05 -0700 Subject: [PATCH 094/133] fix(cmux-tui): key direct echoes from live records --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 113 +++++++++++++++++------ 1 file changed, 85 insertions(+), 28 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 519675ea545f..792c9f8ea214 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6327,34 +6327,49 @@ impl Mux { source: AgentSource, session: Option<&str>, ) -> Option { - let previous = { - let host = self.agent_roster.lock().unwrap(); - let previous = host - .roster - .entries - .get(terminal_id.as_str()) - .map(|entry| { - ( - entry.state.clone(), - entry.source.clone(), - entry.session.clone(), - entry.agent.clone(), - ) - }) - .or_else(|| { - host.roster.last_external_report(terminal_id.as_str()).map( - |(source, state, session)| { - ( - state.to_string(), - source.to_string(), - session.map(str::to_owned), - None, - ) - }, - ) - }); - previous - }; + // The compatibility cache is updated as part of the direct resource + // commit, before its journal echo is folded. Prefer it for admission + // so an unfinished reducer tail cannot make an ABA sequence reuse an + // older idempotency key (working -> blocked -> working). + let previous = self + .agent_records + .lock() + .unwrap() + .get(terminal_id) + .map(|record| { + ( + record.state.as_str().to_string(), + record.source.as_str().to_string(), + record.session.clone(), + record.agent.clone(), + ) + }) + .or_else(|| { + let host = self.agent_roster.lock().unwrap(); + host.roster + .entries + .get(terminal_id.as_str()) + .map(|entry| { + ( + entry.state.clone(), + entry.source.clone(), + entry.session.clone(), + entry.agent.clone(), + ) + }) + .or_else(|| { + host.roster.last_external_report(terminal_id.as_str()).map( + |(source, state, session)| { + ( + state.to_string(), + source.to_string(), + session.map(str::to_owned), + None, + ) + }, + ) + }) + }); let unchanged = previous.as_ref().is_some_and( |(previous_state, previous_source, previous_session, _)| { previous_state == state.as_str() @@ -25948,6 +25963,48 @@ mod tests { ); } + #[test] + fn direct_echo_key_uses_live_cache_when_roster_fold_lags() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + + // Model a blocked reducer tail after the direct projection has already + // accepted a later Working report. Reusing the reducer state here + // would recreate the old Working key and lose the ABA transition. + mux.agent_records.lock().unwrap().insert( + terminal_id.clone(), + TerminalAgentRecord { + state: AgentState::Working, + source: AgentSource::Socket, + session: Some("aba-session".into()), + agent: None, + updated_at_ms: 2, + }, + ); + mux.agent_roster.lock().unwrap().roster.entries.insert( + terminal_id.to_string(), + crate::journal_reducers::RosterEntry { + state: AgentState::Blocked.as_str().into(), + source: AgentSource::Socket.as_str().into(), + session: Some("aba-session".into()), + agent: None, + updated_at_ms: 1, + }, + ); + + assert!( + mux.agent_report_echo_key( + &terminal_id, + AgentState::Working, + AgentSource::Socket, + Some("aba-session"), + ) + .is_none() + ); + mux.shutdown(); + } + #[test] fn agent_reports_require_a_terminal_and_never_create_a_default_projection() { let mux = test_mux(); From 282bd1e6cb20e6b76d6134de1bc166557fe8cab3 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:29:35 -0700 Subject: [PATCH 095/133] fix(cmux-tui): coalesce cross-adapter done echoes --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 792c9f8ea214..96d0fdf21fab 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6377,11 +6377,22 @@ impl Mux { && previous_session.as_deref() == session }, ); + // Done closes a lifecycle independently of which external adapter + // observed it. Coalesce a repeated terminal session even when the + // next poll comes from the other adapter, so the source-neutral fence + // does not accumulate redundant echo rows. + let repeated_external_done = + previous.as_ref().is_some_and(|(previous_state, _, previous_session, _)| { + source != AgentSource::Hook + && state == AgentState::Done + && previous_state == AgentState::Done.as_str() + && previous_session.as_deref() == session + }); let hook_owned_external_report = previous.as_ref().is_some_and(|(_, previous_source, _, _)| { previous_source == AgentSource::Hook.as_str() && source != AgentSource::Hook }); - if unchanged || hook_owned_external_report { + if unchanged || repeated_external_done || hook_owned_external_report { return None; } let previous = previous From 2dae42237bfd87685e7c575c391b77b5a8ee8941 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:32:58 -0700 Subject: [PATCH 096/133] fix(cmux-tui): preserve socket authority over detection --- .../cmux-tui-core/src/journal_reducers.rs | 45 ++++++++++++++++--- cmux-tui/crates/cmux-tui-core/src/mux.rs | 21 ++++++--- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index ff4d0ea86abd..608f88d92e73 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -328,13 +328,15 @@ impl AgentRoster { (state, AgentSource::Hook, public_session, agent, event.committed_at_ms) }; if source != AgentSource::Hook - && self - .entries - .get(terminal_id) - .is_some_and(|entry| entry.agent_source() == AgentSource::Hook) + && self.entries.get(terminal_id).is_some_and(|entry| { + entry.agent_source() == AgentSource::Hook + || (entry.agent_source() == AgentSource::Socket + && source == AgentSource::Detected) + }) { - // Hook state is live agent truth; socket reports cannot - // overwrite it (mirrors the projection commit precedence). + // Hook state is live agent truth. A structured socket report also + // outranks the screen detector, so neither lower-authority source + // can replace a stronger live projection. return Vec::new(); } let external_receipt_changed = if source != AgentSource::Hook { @@ -621,6 +623,37 @@ mod tests { assert_eq!(roster.entries["term_a"].session.as_deref(), Some("screen-1")); } + #[test] + fn detected_echo_cannot_replace_socket_authority() { + let subjects = terminal_subject("term_a"); + let socket = json!({ + "adapter": {"id": SOCKET_REPORT_ADAPTER, "version": 1}, + "normalized": { + "state": "working", + "source": "socket", + "source_session": "socket-1" + } + }); + let detected = json!({ + "adapter": {"id": DETECTED_REPORT_ADAPTER, "version": 1}, + "normalized": { + "state": "blocked", + "source": "detected", + "source_session": "screen-1" + } + }); + let mut roster = AgentRoster::default(); + assert_eq!( + roster.apply(&hook_event(1, "agent.state.changed", &subjects, &socket)).len(), + 1 + ); + assert!( + roster.apply(&hook_event(2, "agent.state.changed", &subjects, &detected)).is_empty() + ); + assert_eq!(roster.entries["term_a"].agent_source(), AgentSource::Socket); + assert_eq!(roster.entries["term_a"].state, AgentState::Working.as_str()); + } + #[test] fn external_done_fence_survives_restart_and_requires_a_new_session() { let subjects = terminal_subject("term_a"); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 96d0fdf21fab..42f9790e33d2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6392,7 +6392,15 @@ impl Mux { previous.as_ref().is_some_and(|(_, previous_source, _, _)| { previous_source == AgentSource::Hook.as_str() && source != AgentSource::Hook }); - if unchanged || repeated_external_done || hook_owned_external_report { + let socket_owned_detected_report = + previous.as_ref().is_some_and(|(_, previous_source, _, _)| { + previous_source == AgentSource::Socket.as_str() && source == AgentSource::Detected + }); + if unchanged + || repeated_external_done + || hook_owned_external_report + || socket_owned_detected_report + { return None; } let previous = previous @@ -10373,14 +10381,15 @@ impl Mux { let now = reported_at_ms.unwrap_or_else(now_ms); let mut records = self.agent_records.lock().unwrap(); let lower_authority_report_ignored = records.get(&terminal_id).is_some_and(|existing| { - existing.source == AgentSource::Hook + (existing.source == AgentSource::Hook && source != AgentSource::Hook - && !effective_hook_state.is_some_and(|state| state.ended) + && !effective_hook_state.is_some_and(|state| state.ended)) + || (existing.source == AgentSource::Socket && source == AgentSource::Detected) }); // Hook-owned state remains a semantic no-op for lower-authority - // detected and socket reports. A socket-owned report still refreshes - // its durable timestamp; the echo key below coalesces only when that - // complete payload is equal. + // reports. Structured socket state also outranks screen detection. + // A socket-owned report still refreshes its durable timestamp; the + // echo key below coalesces only when that complete payload is equal. let record = match records.get(&terminal_id) { Some(existing) if lower_authority_report_ignored => existing.clone(), _ => TerminalAgentRecord { From f13dafd2256c632fc108d7d4af06c7acfd664df9 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:39:12 -0700 Subject: [PATCH 097/133] fix(cmux-tui): enforce report source precedence --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 42f9790e33d2..c00459ca9af8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10419,6 +10419,7 @@ impl Mux { "terminal_id":terminal_id, "state":record.state.as_str(), "source":record.source.as_str(), + "agent":record.agent, "updated_at_ms":record.updated_at_ms.to_string(), "source_session":persisted_source_session.clone().or(record.session.clone()), }); @@ -23538,6 +23539,14 @@ mod tests { let records = mux.list_agents(Some(surface.id), None); assert_eq!(records.len(), 1); assert_eq!(records[0].agent.as_deref(), Some("claude")); + let snapshot = crate::resource_api::public_session_snapshot(&mux).unwrap(); + let public_agent = snapshot["agents"] + .as_array() + .unwrap() + .iter() + .find(|agent| agent["terminal_id"] == terminal_id.as_str()) + .expect("public agent projection"); + assert_eq!(public_agent["agent"], "claude"); let (version, cursor, snapshot) = mux .workspace_registry From c0516ce781c8cbcfc48de5eca9581acd00d396ac Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:58:56 -0700 Subject: [PATCH 098/133] test(cmux-tui): cover ABA echo transitions --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c00459ca9af8..71e5ad968018 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24203,6 +24203,42 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn socket_echoes_get_distinct_keys_after_an_aba_state_cycle() { + let root = std::env::temp_dir() + .join(format!("cmux-roster-aba-echo-{}", crate::workspace_registry::new_uuid_v4())); + let mux = + Mux::open_persistent("roster-aba-echo", SurfaceOptions::default(), &root).unwrap(); + let surface = mux.new_workspace(None, None).unwrap(); + let socket_echoes = || { + mux.session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count() + }; + + for state in + [AgentState::Working, AgentState::Blocked, AgentState::Working, AgentState::Blocked] + { + mux.report_agent(surface.id, state, AgentSource::Socket, Some("aba-session".into())) + .unwrap(); + } + assert_eq!(socket_echoes(), 4, "each semantic ABA transition needs its own echo"); + + mux.shutdown(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn repeated_socket_done_reports_coalesce_after_roster_removal() { let root = std::env::temp_dir() From c10fb49740f8eaf2511a5165ebaf0bb4166d10b2 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:59:41 -0700 Subject: [PATCH 099/133] fix(cmux-tui): make ABA echo keys monotonic --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 71e5ad968018..751b5c7fede5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6327,6 +6327,14 @@ impl Mux { source: AgentSource, session: Option<&str>, ) -> Option { + // Resource revisions are durable and strictly monotonic. Include the + // current revision in a transition key so an ABA sequence cannot + // reuse the key from an earlier cycle after the compatibility cache + // returns to the same state (working -> blocked -> working -> + // blocked). Repeated polls of one semantic state still return early + // through `unchanged`, so unrelated revisions do not create echo + // rows for steady-state reports. + let resource_revision = self.state.lock().unwrap().resource_revision; // The compatibility cache is updated as part of the direct resource // commit, before its journal echo is folded. Prefer it for admission // so an unfinished reducer tail cannot make an ABA sequence reuse an @@ -6412,7 +6420,7 @@ impl Mux { // journal cursor would make an unrelated event defeat coalescing and // turn a steady socket poll into one durable row per event elsewhere. let material = format!( - "agent-report-echo-v2|{terminal_id}|{}|{}|{:?}|{previous}", + "agent-report-echo-v3|{terminal_id}|{}|{}|{:?}|revision={resource_revision}|{previous}", state.as_str(), source.as_str(), session, From d69c98131676ddcdcc37d034d1ff41a6ffd7c367 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:30:10 -0700 Subject: [PATCH 100/133] fix(cmux-tui): bound roster replay retries --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 66 ++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 751b5c7fede5..bba5c8563f16 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1117,6 +1117,19 @@ const AGENT_ROSTER_STARTUP_FOLD_RECORD_LIMIT: usize = 512; /// projection is waiting on an external retry. The timeout is only a wake-up /// bound; SQLite remains the source of truth and no state is inferred from it. const AGENT_ROSTER_FOLD_WORKER_WAIT: Duration = Duration::from_secs(1); +/// Give other journal work a scheduling window between bounded replay pages. +/// This is a fairness delay, not a correctness timer: a new journal epoch +/// wakes the worker immediately. +const AGENT_ROSTER_FOLD_WORKER_BATCH_WAIT: Duration = Duration::from_millis(10); +/// A pending projection already has its own durable retry counter. Stop the +/// reducer worker after one bounded retry budget when that counter cannot +/// release the row, and let a later availability signal or restart retry it. +const AGENT_ROSTER_FOLD_WORKER_PROJECTION_RETRY_LIMIT: u32 = 8; + +fn agent_roster_projection_retry_delay(attempt: u32) -> Duration { + let shift = attempt.saturating_sub(1).min(5); + Duration::from_millis(100u64 << shift) +} #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum AgentRosterFoldProgress { @@ -6090,6 +6103,7 @@ impl Mux { } fn run_agent_roster_fold_worker(mux: Weak) { + let mut projection_retry_attempts = 0u32; loop { let Some(mux) = mux.upgrade() else { return }; if mux.shutting_down.load(Ordering::Acquire) { @@ -6117,17 +6131,58 @@ impl Mux { match mux.fold_agent_roster_through(target_sequence) { AgentRosterFoldProgress::MoreAvailable => { // Keep each pass bounded and yield between pages so a - // large restored journal cannot monopolize a core. - std::thread::yield_now(); + // large restored journal cannot monopolize a core. Wait + // on the journal signal rather than spinning; a commit + // wakes this immediately while an old backlog advances + // at a bounded cadence. + projection_retry_attempts = 0; + let epoch = mux.journal_event_epoch(); + mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_BATCH_WAIT); } AgentRosterFoldProgress::WaitingForProjection => { + // Retry the durable projection receipt. Each failed + // attempt advances its persistent counter and eventually + // quarantines an unresolvable row, which lets the + // reducer move past it instead of polling forever. + projection_retry_attempts = projection_retry_attempts.saturating_add(1); + if let Err(error) = mux.retry_pending_agent_hooks() { + eprintln!( + "cmux-tui: retrying pending agent projections from the roster worker failed: {error}" + ); + } + if projection_retry_attempts >= AGENT_ROSTER_FOLD_WORKER_PROJECTION_RETRY_LIMIT + { + // Give the retry one final bounded fold pass. A + // successful quarantine or cleanup clears the + // counter and continues; only a still-pending row + // stops the worker and reports recovery state. + match mux.fold_agent_roster_through(target_sequence) { + AgentRosterFoldProgress::ReachedTarget + | AgentRosterFoldProgress::MoreAvailable => { + projection_retry_attempts = 0; + continue; + } + AgentRosterFoldProgress::WaitingForProjection => { + mux.report_internal_diagnostic( + "agent roster replay paused after bounded projection retries; a later terminal-availability signal or restart will retry the pending event", + ); + mux.mark_agent_roster_fold_worker_stopped(); + return; + } + } + } // A retry may clear the pending projection without - // appending a journal row. The bounded timeout handles - // that case while the epoch handles normal commits. + // appending a journal row. The bounded exponential delay + // handles repeated failures while the epoch handles + // normal commits and successful cleanup wakes promptly. let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_WAIT); + mux.wait_for_journal_event( + epoch, + agent_roster_projection_retry_delay(projection_retry_attempts), + ); } AgentRosterFoldProgress::ReachedTarget => { + projection_retry_attempts = 0; // Close the worker only after a race-safe epoch/head // handshake. A commit that arrives before the flag is // cleared is either visible in the head or changes the @@ -23627,6 +23682,7 @@ mod tests { records.get(&terminal_id).and_then(|record| record.agent.as_deref()), Some("claude") ); + drop(records); reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); From ff0d010211599578eb01aa604c0007c406efc1a0 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:55:27 -0700 Subject: [PATCH 101/133] test(cmux-tui): cover fresh external lifecycle after hook end --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bba5c8563f16..3dc802e94101 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -25243,6 +25243,20 @@ mod tests { }; mux.append_journal_ingress(&hook("SessionEnd", "old-hook"), "test", "ended-hook").unwrap(); + // Simulate a compatibility cache that outlives the durable end + // fence while bounded startup replay or cleanup is still pending. + // A proven new socket lifecycle must replace this stale Hook record. + mux.agent_records.lock().unwrap().insert( + terminal_id.clone(), + TerminalAgentRecord { + state: AgentState::Idle, + source: AgentSource::Hook, + session: Some("old-hook".into()), + agent: Some("claude".into()), + updated_at_ms: 1, + }, + ); + let record = mux .report_agent( surface.id, From 155111b9d3b60c432cd3be1707d0faa6973b530a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:56:02 -0700 Subject: [PATCH 102/133] fix(cmux-tui): bound roster projection checks --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 54 +++++++++++++------ .../src/workspace_registry/resource_store.rs | 9 ++-- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 3dc802e94101..6987a0a1993e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1182,8 +1182,18 @@ fn restore_agent_roster(registry: &WorkspaceRegistry) -> anyhow::Result sequences, - Err(error) => { - eprintln!( - "cmux-tui: reading pending agent hook projections before roster fold failed: {error}" - ); - return AgentRosterFoldProgress::WaitingForProjection; - } - }; let mut folded_records = 0usize; loop { let page = match self @@ -5937,6 +5933,26 @@ impl Mux { return AgentRosterFoldProgress::WaitingForProjection; } }; + let page_through_sequence = page + .records + .last() + .map(|record| record.sequence) + .unwrap_or(read_cursor) + .min(target_sequence); + let (pending_sequences, quarantined_sequences) = match self + .workspace_registry + .lock() + .unwrap() + .agent_hook_projection_sequence_states(read_cursor, page_through_sequence) + { + Ok(sequences) => sequences, + Err(error) => { + eprintln!( + "cmux-tui: reading pending agent hook projections before roster fold failed: {error}" + ); + return AgentRosterFoldProgress::WaitingForProjection; + } + }; let mut advanced = false; let mut semantic_change = false; let mut removal = false; @@ -10357,6 +10373,7 @@ impl Mux { .cloned(); let ended_fence = current_fence.as_ref().filter(|fence| fence.ended); let mut direct_hook_state = None; + let mut external_lifecycle_reopened = false; if source != AgentSource::Hook { let new_lifecycle_session = source_session.as_deref().is_some_and(|session| { !session.is_empty() @@ -10366,6 +10383,12 @@ impl Mux { && !durable_hook_session.as_deref().is_some_and(|old| session == old) && !durable_external_session.as_deref().is_some_and(|old| session == old) }); + // A new external session may replace a stale Hook compatibility + // record only when the durable hook fence proves that the old + // lifecycle ended. A merely different session must still remain + // below an active Hook record. + external_lifecycle_reopened = + new_lifecycle_session && (durable_hook_session.is_some() || ended_fence.is_some()); if (durable_hook_session.is_some() || durable_external_end || ended_fence.is_some()) && !new_lifecycle_session && !(durable_external_end @@ -10446,6 +10469,7 @@ impl Mux { let lower_authority_report_ignored = records.get(&terminal_id).is_some_and(|existing| { (existing.source == AgentSource::Hook && source != AgentSource::Hook + && !external_lifecycle_reopened && !effective_hook_state.is_some_and(|state| state.ended)) || (existing.source == AgentSource::Socket && source == AgentSource::Detected) }); diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 554cf6755022..01c6c29a3ed0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -256,7 +256,9 @@ pub(super) fn create_resource_schema(transaction: &Transaction<'_>) -> anyhow::R transaction.execute_batch( "DROP INDEX IF EXISTS resource_agent_hook_pending_by_terminal; CREATE INDEX IF NOT EXISTS resource_agent_hook_pending_by_terminal - ON resource_agent_hook_pending(terminal_id, event_sequence, idempotency_key);", + ON resource_agent_hook_pending(terminal_id, event_sequence, idempotency_key); + CREATE INDEX IF NOT EXISTS resource_agent_hook_pending_by_sequence + ON resource_agent_hook_pending(event_sequence, attempt);", )?; Ok(()) } @@ -770,9 +772,10 @@ impl WorkspaceRegistry { return Ok((HashSet::new(), HashSet::new())); } let mut statement = self.connection.prepare( - "SELECT event_sequence, attempt + "SELECT event_sequence, MAX(attempt) FROM resource_agent_hook_pending - WHERE event_sequence > ?1 AND event_sequence <= ?2", + WHERE event_sequence > ?1 AND event_sequence <= ?2 + GROUP BY event_sequence", )?; let mut pending = HashSet::new(); let mut quarantined = HashSet::new(); From 8686d8fca7ac6b9e34b072ce60ce73f770ae8550 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:08:49 -0700 Subject: [PATCH 103/133] fix(cmux-tui): bound roster shutdown recovery --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 71 +++++++++++++++++++----- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 6987a0a1993e..18a3b86dec80 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1085,6 +1085,16 @@ impl AgentSource { AgentSource::Hook => "hook", } } + + /// Higher values have stronger authority when two projections describe + /// the same terminal lifecycle. + pub(crate) const fn authority_rank(self) -> u8 { + match self { + AgentSource::Detected => 0, + AgentSource::Socket => 1, + AgentSource::Hook => 2, + } + } } /// The journal-derived agent roster and the last journal sequence folded into @@ -1125,6 +1135,10 @@ const AGENT_ROSTER_FOLD_WORKER_BATCH_WAIT: Duration = Duration::from_millis(10); /// reducer worker after one bounded retry budget when that counter cannot /// release the row, and let a later availability signal or restart retry it. const AGENT_ROSTER_FOLD_WORKER_PROJECTION_RETRY_LIMIT: u32 = 8; +/// Do not keep a detached reducer worker alive forever when the journal +/// remains unavailable. A later journal event or startup can schedule a new +/// worker after this bounded recovery state is reported. +const AGENT_ROSTER_FOLD_WORKER_JOURNAL_RETRY_LIMIT: u32 = 8; fn agent_roster_projection_retry_delay(attempt: u32) -> Duration { let shift = attempt.saturating_sub(1).min(5); @@ -6120,6 +6134,7 @@ impl Mux { fn run_agent_roster_fold_worker(mux: Weak) { let mut projection_retry_attempts = 0u32; + let mut journal_retry_attempts = 0u32; loop { let Some(mux) = mux.upgrade() else { return }; if mux.shutting_down.load(Ordering::Acquire) { @@ -6133,13 +6148,27 @@ impl Mux { .unwrap() .session_journal_after(0, 1) { - Ok(page) => page.head_sequence, + Ok(page) => { + journal_retry_attempts = 0; + page.head_sequence + } Err(error) => { eprintln!( "cmux-tui: reading the agent roster journal head in the worker failed: {error}" ); + journal_retry_attempts = journal_retry_attempts.saturating_add(1); + if journal_retry_attempts >= AGENT_ROSTER_FOLD_WORKER_JOURNAL_RETRY_LIMIT { + mux.report_internal_diagnostic( + "agent roster replay paused after bounded journal read retries; a later journal event or restart will retry the pending events", + ); + mux.mark_agent_roster_fold_worker_stopped(); + return; + } let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_WAIT); + mux.wait_for_journal_event( + epoch, + agent_roster_projection_retry_delay(journal_retry_attempts), + ); continue; } }; @@ -6362,17 +6391,19 @@ impl Mux { match delta { RosterDelta::Upsert { terminal_id, entry } => { let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) else { return }; - if records - .get(&terminal_id) - .is_some_and(|existing| existing.updated_at_ms > entry.updated_at_ms) - { + let source = entry.agent_source(); + if records.get(&terminal_id).is_some_and(|existing| { + existing.source.authority_rank() > source.authority_rank() + || (existing.source == source + && existing.updated_at_ms > entry.updated_at_ms) + }) { return; } records.insert( terminal_id, TerminalAgentRecord { state: entry.agent_state(), - source: entry.agent_source(), + source, session: entry.session, agent: entry.agent, updated_at_ms: entry.updated_at_ms, @@ -10800,7 +10831,7 @@ impl Mux { eprintln!("cmux-tui: journal hook workers did not stop before the shutdown deadline"); } self.finalize_terminal_journal("shutdown"); - if let Err(error) = self.persist_agent_roster_snapshot() { + if let Err(error) = self.persist_agent_roster_snapshot_until(hook_deadline) { eprintln!( "cmux-tui: persisting the agent roster snapshot during shutdown failed: {error}" ); @@ -10813,22 +10844,36 @@ impl Mux { /// Flush a coalesced roster cursor before the daemon exits. The journal is /// already closed by the caller, so this write observes the final fold. - fn persist_agent_roster_snapshot(&self) -> anyhow::Result<()> { - let _fold = self.agent_roster_fold.lock().unwrap(); + fn persist_agent_roster_snapshot_until(&self, deadline: Instant) -> anyhow::Result<()> { + // Shutdown must never block on a worker that missed its deadline. + // `agent_roster_fold` is a plain mutex, so use its non-blocking path; + // the registry mutex has a deadline-aware lock implementation. + let _fold = self + .agent_roster_fold + .try_lock() + .map_err(|_| anyhow::anyhow!("agent roster fold lock unavailable at shutdown"))?; let (cursor, snapshot) = { - let host = self.agent_roster.lock().unwrap(); + let host = self + .agent_roster + .try_lock() + .map_err(|_| anyhow::anyhow!("agent roster state lock unavailable at shutdown"))?; if host.cursor == host.persisted_cursor { return Ok(()); } (host.cursor, host.roster.snapshot().to_string()) }; - self.workspace_registry.lock().unwrap().put_journal_reducer_state( + self.workspace_registry.lock_until(deadline)?.put_journal_reducer_state( crate::journal_reducers::AGENT_ROSTER_REDUCER_ID, crate::journal_reducers::AGENT_ROSTER_REDUCER_VERSION, cursor, &snapshot, )?; - self.agent_roster.lock().unwrap().persisted_cursor = cursor; + let mut host = self.agent_roster.try_lock().map_err(|_| { + anyhow::anyhow!("agent roster state lock unavailable after shutdown persist") + })?; + if host.cursor == cursor { + host.persisted_cursor = cursor; + } Ok(()) } From a7d2d3464d32b0e878804b336d724b1be8c914e8 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:35:36 -0700 Subject: [PATCH 104/133] fix(cmux-tui): restore durable agent identity --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 8 ++++++++ .../crates/cmux-tui-core/src/mux/public_projections.rs | 6 +++++- .../src/workspace_registry/public_projection_store.rs | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 18a3b86dec80..92c52f724eca 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6480,6 +6480,14 @@ impl Mux { ) }) }); + #[cfg(test)] + if session == Some("repeat-session") { + eprintln!( + "agent-report-echo diagnostic: previous={previous:?} state={} source={} session={session:?} revision={resource_revision}", + state.as_str(), + source.as_str(), + ); + } let unchanged = previous.as_ref().is_some_and( |(previous_state, previous_source, previous_session, _)| { previous_state == state.as_str() diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs index 7ad08f24e203..6f7d666bae97 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs @@ -110,7 +110,7 @@ pub(super) fn restore_public_projections( state, source: agent_source(&agent.source)?, session: (!internal_marker).then_some(agent.source_session).flatten(), - agent: None, + agent: agent.agent, updated_at_ms: agent.updated_at_ms, }, ); @@ -224,6 +224,7 @@ mod tests { terminal_id: terminal.clone(), state: "working".into(), source: "hook".into(), + agent: None, updated_at_ms: 1, source_session: None, }], @@ -302,6 +303,7 @@ mod tests { terminal_id: terminal.clone(), state: "done".into(), source: "hook".into(), + agent: None, updated_at_ms: 1, source_session: None, }], @@ -324,6 +326,7 @@ mod tests { terminal_id: terminal.clone(), state: "working".into(), source: "hook".into(), + agent: None, updated_at_ms: 1, source_session: Some("cmux-hook-sequence:12".into()), }], @@ -346,6 +349,7 @@ mod tests { terminal_id: terminal.clone(), state: "done".into(), source: "socket".into(), + agent: None, updated_at_ms: 3, source_session: Some("socket-session".into()), }], diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs index 2555de3cccfb..7cf9aff49c08 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs @@ -40,6 +40,7 @@ pub struct RegistryAgentProjection { pub terminal_id: TerminalPublicId, pub state: String, pub source: String, + pub agent: Option, pub updated_at_ms: u64, pub source_session: Option, } @@ -60,6 +61,7 @@ impl RegistryAgentProjection { "terminal_id": self.terminal_id, "state": self.state, "source": self.source, + "agent": self.agent, "updated_at_ms": self.updated_at_ms.to_string(), "source_session": self.source_session, }) @@ -117,6 +119,8 @@ struct StoredAgent { terminal_id: TerminalPublicId, state: StoredAgentState, source: StoredAgentSource, + #[serde(default)] + agent: Option, updated_at_ms: WireDecimal, source_session: Option, #[serde(default)] @@ -356,6 +360,7 @@ impl WorkspaceRegistry { terminal_id: stored.terminal_id, state: stored.state.as_str().to_string(), source: stored.source.as_str().to_string(), + agent: stored.agent, updated_at_ms: stored.updated_at_ms.get(), source_session: stored.source_session, }); From b6d1c3cce540ecd158324dc991c95ddd13647a9e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:46:51 -0700 Subject: [PATCH 105/133] test(cmux-tui): quarantine missing terminal events --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 55 ++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 92c52f724eca..20a2ab5ddf46 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24119,6 +24119,61 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn missing_terminal_hook_event_is_quarantined_before_roster_replay() { + let root = std::env::temp_dir().join(format!( + "cmux-roster-missing-terminal-{}", + crate::workspace_registry::new_uuid_v4() + )); + let session = "roster-missing-terminal"; + let terminal_id = + TerminalPublicId::parse("term_ffffffffffffffffffffffffffffffff").unwrap(); + let ingress = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&terminal_id.to_string()), + serde_json::json!({"session_id":"missing-terminal"}), + ) + .unwrap(); + let mux = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); + let commit = mux.append_journal_ingress(&ingress, "test", "missing-terminal").unwrap(); + assert!( + mux.workspace_registry + .lock() + .unwrap() + .agent_hook_pending_is_quarantined( + &ingress.producer_id, + "test", + "missing-terminal", + ) + .unwrap(), + "a missing terminal must leave a durable reducer skip marker" + ); + mux.shutdown(); + drop(mux); + + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + let reopened = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let roster = reopened.agent_roster.lock().unwrap(); + assert!( + roster.roster.entries.is_empty(), + "missing terminal event at sequence {} must not create ghost roster state", + commit.sequence + ); + assert!(roster.cursor >= commit.sequence); + drop(roster); + reopened.shutdown(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn detected_report_cannot_replace_hook_owned_state() { let mux = test_mux(); From bf5ff610dbc1c45cd17cf2746eb59a0426d4992b Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:47:43 -0700 Subject: [PATCH 106/133] fix(cmux-tui): quarantine gone terminal events --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 35 ++++++++++----- .../src/workspace_registry/resource_store.rs | 45 +++++++++++++++++++ 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 20a2ab5ddf46..ed1bb95b47d2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2947,12 +2947,19 @@ impl Mux { .workspace_registry .lock() .unwrap() - .clear_agent_hook_pending(&producer_id, &origin, &key) + .quarantine_agent_hook_pending( + &producer_id, + &origin, + &key, + sequence, + &ingress, + &error.to_string(), + ) .is_ok() { applied += 1; - // Removing a pending marker can release a contiguous - // roster prefix even when the terminal is gone. Keep + // The quarantined marker releases a contiguous roster + // prefix without applying the impossible event. Keep // one representative ingress so the reducer is // retried after the durable skip is recorded. fold_ingress = Some(ingress); @@ -5831,15 +5838,24 @@ impl Mux { { should_fold_roster = false; if agent_hook_terminal_gone(&error) { - if let Err(_bookkeeping_error) = self - .workspace_registry - .lock() - .unwrap() - .clear_agent_hook_pending(&ingress.producer_id, origin, idempotency_key) + if let Err(_bookkeeping_error) = + self.workspace_registry.lock().unwrap().quarantine_agent_hook_pending( + &ingress.producer_id, + origin, + idempotency_key, + commit.sequence, + ingress, + &error.to_string(), + ) { self.report_internal_diagnostic( "terminal-gone agent hook cleanup deferred", ); + } else { + // The durable marker is an explicit reducer skip. It + // must be folded so the contiguous journal cursor can + // advance past this impossible event. + should_fold_roster = true; } } else { if let Err(_bookkeeping_error) = @@ -24126,8 +24142,7 @@ mod tests { crate::workspace_registry::new_uuid_v4() )); let session = "roster-missing-terminal"; - let terminal_id = - TerminalPublicId::parse("term_ffffffffffffffffffffffffffffffff").unwrap(); + let terminal_id = TerminalPublicId::parse("term_ffffffffffffffffffffffffffffffff").unwrap(); let ingress = crate::agent_hooks::agent_hook_journal_ingress( "claude", "UserPromptSubmit", diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 01c6c29a3ed0..508dba10dc46 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -658,6 +658,51 @@ impl WorkspaceRegistry { Ok(()) } + /// Keep a durable dead-letter marker when a journal event names a terminal + /// that can never be materialized. The reducer must see this marker and + /// skip the event; deleting it would let a later replay create ghost + /// roster state. + pub(crate) fn quarantine_agent_hook_pending( + &mut self, + producer_id: &str, + origin: &str, + idempotency_key: &str, + sequence: u64, + ingress: &crate::JournalIngress, + error: &str, + ) -> anyhow::Result<()> { + const MAX_ERROR_CHARS: usize = 1_024; + let ingress_json = serde_json::to_string(ingress)?; + let terminal_id = ingress + .subjects + .iter() + .find(|subject| subject.kind == "terminal") + .map(|subject| subject.id.as_str()); + let bounded_error = error.chars().take(MAX_ERROR_CHARS).collect::(); + self.connection.execute( + "INSERT INTO resource_agent_hook_pending( + producer_id, origin, idempotency_key, terminal_id, event_sequence, ingress_json, error, attempt + ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(producer_id, origin, idempotency_key) DO UPDATE SET + terminal_id = excluded.terminal_id, + event_sequence = excluded.event_sequence, + ingress_json = excluded.ingress_json, + error = excluded.error, + attempt = MAX(resource_agent_hook_pending.attempt, excluded.attempt)", + params![ + producer_id, + origin, + idempotency_key, + terminal_id, + i64::try_from(sequence)?, + ingress_json, + bounded_error, + AGENT_HOOK_MAX_ATTEMPTS, + ], + )?; + Ok(()) + } + pub(crate) fn purge_agent_hook_pending_for_terminal( &mut self, terminal_id: &crate::resource::TerminalPublicId, From 4734aa0e2a93141e5de03388159e5ddded335bec Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:26:05 -0700 Subject: [PATCH 107/133] test(cmux-tui): separate resource refresh from echo coalescing --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 39 ++++++++++++++++++------ 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ed1bb95b47d2..85cbec49cdda 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -26207,6 +26207,24 @@ mod tests { fn socket_agent_report_folds_roster_and_coalesces_repeated_state() { let mux = test_mux(); let surface = mux.new_workspace(None, None).unwrap(); + let socket_echoes = || { + mux.workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count() + }; let before_cursor = mux.agent_roster.lock().unwrap().cursor; let before_head = mux .workspace_registry @@ -26233,6 +26251,7 @@ mod tests { .head_sequence; assert!(after_first_cursor > before_cursor); assert!(after_first_head > before_head); + assert_eq!(socket_echoes(), 1); mux.report_agent( surface.id, @@ -26242,15 +26261,17 @@ mod tests { ) .unwrap(); assert_eq!(mux.agent_roster.lock().unwrap().cursor, after_first_cursor); - assert_eq!( - mux.workspace_registry - .lock() - .unwrap() - .session_journal_after(0, 1) - .unwrap() - .head_sequence, - after_first_head - ); + // The direct projection still persists a fresh timestamp, which is + // a resource journal record. Only the agent echo is coalesced. + let after_second_head = mux + .workspace_registry + .lock() + .unwrap() + .session_journal_after(0, 1) + .unwrap() + .head_sequence; + assert!(after_second_head > after_first_head); + assert_eq!(socket_echoes(), 1, "the repeated state has one agent echo"); } #[test] From 9b2eb83b5b39a4f0749d19af514e1e279d3cc027 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:26:31 -0700 Subject: [PATCH 108/133] chore(cmux-tui): remove temporary roster diagnostic --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 85cbec49cdda..a081de96d1f5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6496,14 +6496,6 @@ impl Mux { ) }) }); - #[cfg(test)] - if session == Some("repeat-session") { - eprintln!( - "agent-report-echo diagnostic: previous={previous:?} state={} source={} session={session:?} revision={resource_revision}", - state.as_str(), - source.as_str(), - ); - } let unchanged = previous.as_ref().is_some_and( |(previous_state, previous_source, previous_session, _)| { previous_state == state.as_str() From 7187fac186356724e34aee06ead70d53d854b629 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:07:24 -0700 Subject: [PATCH 109/133] test(cmux-tui): cover lifecycle fence races --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 88 ++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a081de96d1f5..f26fc1435be2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23876,6 +23876,85 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[test] + fn delayed_roster_remove_cannot_delete_a_newer_lifecycle_record() { + use crate::journal_reducers::RosterDelta; + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("terminal"); + mux.agent_records.lock().unwrap().insert( + terminal_id.clone(), + TerminalAgentRecord { + state: AgentState::Working, + source: AgentSource::Socket, + session: Some("new-session".into()), + agent: None, + updated_at_ms: 200, + }, + ); + + // A delayed Done fold belongs to the previous lifecycle. It must not + // remove the newer direct record that now occupies the same terminal. + mux.apply_roster_delta_to_record_cache(RosterDelta::Remove { + terminal_id: terminal_id.to_string(), + }); + + let record = mux.agent_records.lock().unwrap(); + assert_eq!( + record.get(&terminal_id).and_then(|entry| entry.session.as_deref()), + Some("new-session") + ); + mux.shutdown(); + } + + #[test] + fn direct_done_report_fences_same_session_before_roster_fold() { + let mux = test_mux(); + let blocked_surface = mux.new_workspace(None, None).unwrap(); + let target_surface = mux.new_workspace(None, None).unwrap(); + let blocked_terminal = blocked_surface.terminal_public_id().cloned().expect("terminal"); + + // Leave an earlier hook projection pending. The direct Done echo for + // the target terminal commits, but its roster fold must stop behind + // this unresolved prefix. + mux.remove_terminal_catalog_for_test(&blocked_terminal); + let pending_hook = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&blocked_terminal.to_string()), + serde_json::json!({}), + ) + .unwrap(); + let validated = mux.journal_kernel.validate_ingress(&pending_hook).unwrap(); + mux.workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&pending_hook, &validated, "test", "done-fence-prefix") + .unwrap(); + + let done = mux + .report_agent( + target_surface.id, + AgentState::Done, + AgentSource::Socket, + Some("direct-session".into()), + ) + .unwrap(); + assert_eq!(done.state, AgentState::Done); + + let working = mux.report_agent( + target_surface.id, + AgentState::Working, + AgentSource::Socket, + Some("direct-session".into()), + ); + assert!( + working.is_err(), + "a same-session Working report must respect the durable Done fence" + ); + mux.shutdown(); + } + #[test] fn roster_fold_pages_to_a_target_beyond_the_first_journal_page() { let root = std::env::temp_dir().join(format!( @@ -26080,8 +26159,8 @@ mod tests { let failed = mux.purge_terminal_side_tables(&terminal_id); assert!(failed.is_err(), "the durable snapshot error must remain observable"); assert!( - !mux.agent_records.lock().unwrap().contains_key(&terminal_id), - "a failed roster snapshot must not retain the compatibility record" + mux.agent_records.lock().unwrap().contains_key(&terminal_id), + "a failed roster snapshot must retain the compatibility record" ); assert!( mux.workspace_registry @@ -26089,8 +26168,9 @@ mod tests { .unwrap() .pending_agent_hook_projections() .unwrap() - .is_empty(), - "durable pending hook receipts must still be purged" + .iter() + .any(|(_, _, key, _, _)| key == "pending-before-snapshot-failure"), + "durable pending hook receipts must remain until the fence is durable" ); assert!( !mux.agent_roster.lock().unwrap().roster.is_retired(terminal_id.as_str()), From cd581a25f9d2a0f05dfc809f6cddb3b9d4c772e5 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:11:11 -0700 Subject: [PATCH 110/133] fix(cmux-tui): fence delayed roster lifecycles --- .../cmux-tui-core/src/journal_reducers.rs | 55 ++++++++++--- cmux-tui/crates/cmux-tui-core/src/mux.rs | 81 +++++++++++++++---- 2 files changed, 110 insertions(+), 26 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 608f88d92e73..1bc0d265475e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -176,8 +176,17 @@ impl RosterEntry { /// only mutates roster state. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum RosterDelta { - Upsert { terminal_id: String, entry: RosterEntry }, - Remove { terminal_id: String }, + Upsert { + terminal_id: String, + entry: RosterEntry, + }, + /// Remove only the lifecycle represented by `entry`. A fold may lag a + /// newer direct projection, so the host must not delete that newer record + /// just because the terminal id matches. + Remove { + terminal_id: String, + entry: RosterEntry, + }, } /// Live-agent roster: terminal public id to the agent's last reported @@ -362,11 +371,18 @@ impl AgentRoster { // An ended agent leaves the roster entirely; the done state is // still committed to the durable projection by the host so // history and remote caches converge. - return if self.entries.remove(terminal_id).is_some() - || source == AgentSource::Hook - || external_receipt_changed - { - vec![RosterDelta::Remove { terminal_id: terminal_id.to_string() }] + let removed = self.entries.remove(terminal_id); + return if removed.is_some() || source == AgentSource::Hook || external_receipt_changed { + vec![RosterDelta::Remove { + terminal_id: terminal_id.to_string(), + entry: RosterEntry { + state: state.as_str().to_string(), + source: source.as_str().to_string(), + session, + agent, + updated_at_ms, + }, + }] } else { Vec::new() }; @@ -553,7 +569,19 @@ mod tests { assert_eq!(roster.entries["term_a"].state, "blocked"); let deltas = roster.apply(&hook_event(5, "agent.session.ended", &subjects, &payload)); - assert_eq!(deltas, vec![RosterDelta::Remove { terminal_id: "term_a".into() }]); + assert_eq!( + deltas, + vec![RosterDelta::Remove { + terminal_id: "term_a".into(), + entry: RosterEntry { + state: "done".into(), + source: "hook".into(), + session: None, + agent: None, + updated_at_ms: 1_005, + }, + }] + ); assert!(roster.entries.is_empty()); } @@ -669,7 +697,16 @@ mod tests { assert_eq!( roster.apply(&hook_event(1, "agent.state.changed", &subjects, &done)), - vec![RosterDelta::Remove { terminal_id: "term_a".into() }] + vec![RosterDelta::Remove { + terminal_id: "term_a".into(), + entry: RosterEntry { + state: "done".into(), + source: "detected".into(), + session: Some("screen-1".into()), + agent: None, + updated_at_ms: 1_001, + }, + }] ); assert!(roster.entries.is_empty()); assert!(roster.has_terminal_removal_fence("term_a")); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f26fc1435be2..dd00127bcb34 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6426,9 +6426,20 @@ impl Mux { }, ); } - RosterDelta::Remove { terminal_id } => { + RosterDelta::Remove { terminal_id, entry } => { if let Ok(terminal_id) = TerminalPublicId::parse(terminal_id) { - records.remove(&terminal_id); + // A delayed reducer row may describe an ended lifecycle + // while a newer direct report already occupies the same + // terminal. Remove only when source, session, and the + // direct timestamp still identify the ended lifecycle. + let remove = records.get(&terminal_id).is_some_and(|existing| { + existing.source.as_str() == entry.source + && existing.session == entry.session + && existing.updated_at_ms <= entry.updated_at_ms + }); + if remove { + records.remove(&terminal_id); + } } } } @@ -10396,21 +10407,37 @@ impl Mux { (surface, (*terminal_id).clone()) } }; - let (durable_hook_session, durable_external_session, durable_external_end) = { + // Read the roster fence and the authoritative resource projection + // while the registry lock is held. The reducer can lag a committed + // direct echo behind an unrelated pending row, so the projection is + // the durable admission fence until that echo is folded. + let durable_external_projection = registry + .public_agent_projections(Some(&terminal_id), None)? + .into_iter() + .find(|agent| { + agent.source != AgentSource::Hook.as_str() + && agent.state == AgentState::Done.as_str() + }); + let (durable_hook_session, roster_external_session, roster_external_end) = { let roster = self.agent_roster.lock().unwrap(); if roster.roster.is_retired(terminal_id.as_str()) { anyhow::bail!("terminal {terminal_id} has been retired"); } let durable_hook_session = roster.roster.ended_hook_session(terminal_id.as_str()).map(str::to_owned); - let durable_external_session = + let roster_external_session = roster.roster.ended_external_session(terminal_id.as_str()).map(str::to_owned); - let durable_external_end = roster + let roster_external_end = roster .roster .last_external_report(terminal_id.as_str()) .is_some_and(|(_, state, _)| state == AgentState::Done.as_str()); - (durable_hook_session, durable_external_session, durable_external_end) + (durable_hook_session, roster_external_session, roster_external_end) }; + let durable_external_session = durable_external_projection + .as_ref() + .and_then(|agent| agent.source_session.clone()) + .or(roster_external_session); + let durable_external_end = roster_external_end || durable_external_projection.is_some(); // A caller-held fence is just as authoritative as one acquired in // this function. Clone the entry before any later mutable insertion // so the validation decision cannot race or borrow the guard. @@ -10711,10 +10738,9 @@ impl Mux { // persisted before the compatibility cache is cleared. let mut fences = self.agent_hook_fences.lock().unwrap(); let _fold = self.agent_roster_fold.lock().unwrap(); - let (journal_cursor, pending_cleanup) = { - let mut registry = self.workspace_registry.lock().unwrap(); - let pending_cleanup = registry.purge_agent_hook_pending_for_terminal(terminal_id); - let journal_cursor = match registry.session_journal_after(0, 1) { + let journal_cursor = { + let registry = self.workspace_registry.lock().unwrap(); + match registry.session_journal_after(0, 1) { Ok(page) => page.head_sequence, Err(error) => { self.report_internal_diagnostic( @@ -10722,13 +10748,8 @@ impl Mux { ); return Err(error).context("read journal head before terminal retirement"); } - }; - (journal_cursor, pending_cleanup) + } }; - let pending_cleanup_error = pending_cleanup.err().map(|error| { - self.report_internal_diagnostic("terminal agent hook cleanup deferred"); - error.context("purge pending agent hook receipts") - }); let retired = { let mut host = self.agent_roster.lock().unwrap(); let previous = host.roster.clone(); @@ -10757,12 +10778,31 @@ impl Mux { } else { None }; + if let Some(error) = snapshot_error { + // Keep every live and pending side table intact until the + // retirement fence is durable. Removing pending receipts first + // would let an old journal row be projected again after a + // transient snapshot failure. + drop(_fold); + drop(fences); + return Err(error); + } + let pending_cleanup_error = self + .workspace_registry + .lock() + .unwrap() + .purge_agent_hook_pending_for_terminal(terminal_id) + .err() + .map(|error| error.context("purge pending agent hook receipts")); fences.remove(terminal_id); drop(_fold); drop(fences); self.agent_records.lock().unwrap().remove(terminal_id); self.terminal_notifications.lock().unwrap().remove(terminal_id); - snapshot_error.or(pending_cleanup_error).map_or(Ok(()), Err) + if pending_cleanup_error.is_some() { + self.report_internal_diagnostic("terminal agent hook cleanup deferred"); + } + pending_cleanup_error.map_or(Ok(()), Err) } fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) -> anyhow::Result<()> { @@ -23897,6 +23937,13 @@ mod tests { // remove the newer direct record that now occupies the same terminal. mux.apply_roster_delta_to_record_cache(RosterDelta::Remove { terminal_id: terminal_id.to_string(), + entry: crate::journal_reducers::RosterEntry { + state: "done".into(), + source: "socket".into(), + session: Some("old-session".into()), + agent: None, + updated_at_ms: 100, + }, }); let record = mux.agent_records.lock().unwrap(); From 5806746864301a855942aa5d602bf1ff1dcaf7be Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:13:53 -0700 Subject: [PATCH 111/133] test(cmux-tui): hold roster worker during fence race --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index dd00127bcb34..4d9c4826bd10 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23979,6 +23979,10 @@ mod tests { .append_journal_ingress(&pending_hook, &validated, "test", "done-fence-prefix") .unwrap(); + // Keep the test's unresolved prefix in place while the direct report + // exercises its admission fence. The normal worker is tested by the + // dedicated retry cases below. + mux.agent_roster_fold_worker_running.store(true, Ordering::Release); let done = mux .report_agent( target_surface.id, @@ -23999,6 +24003,7 @@ mod tests { working.is_err(), "a same-session Working report must respect the durable Done fence" ); + mux.agent_roster_fold_worker_running.store(false, Ordering::Release); mux.shutdown(); } From 83ebc4b130f27d6554aa75e1fac8bbed38a93cbd Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:34:56 -0700 Subject: [PATCH 112/133] test(cmux-tui): bound done echoes during blocked roster replay --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 57 ++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4d9c4826bd10..c74570d80bde 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24007,6 +24007,63 @@ mod tests { mux.shutdown(); } + #[test] + fn repeated_socket_done_reports_coalesce_when_roster_replay_is_blocked() { + let mux = test_mux(); + let blocked_surface = mux.new_workspace(None, None).unwrap(); + let target_surface = mux.new_workspace(None, None).unwrap(); + let blocked_terminal = blocked_surface.terminal_public_id().cloned().expect("terminal"); + + // An earlier unresolved projection prevents the roster from folding + // the target echo. Repeated direct Done reports must still use the + // durable projection as their idempotency fence. + mux.remove_terminal_catalog_for_test(&blocked_terminal); + let pending_hook = crate::agent_hooks::agent_hook_journal_ingress( + "claude", + "UserPromptSubmit", + Some(&blocked_terminal.to_string()), + serde_json::json!({}), + ) + .unwrap(); + let validated = mux.journal_kernel.validate_ingress(&pending_hook).unwrap(); + mux.workspace_registry + .lock() + .unwrap() + .append_journal_ingress(&pending_hook, &validated, "test", "done-echo-prefix") + .unwrap(); + mux.agent_roster_fold_worker_running.store(true, Ordering::Release); + + let socket_echoes = || { + mux.session_journal_after(0, 1024) + .unwrap() + .records + .into_iter() + .filter(|record| { + record + .payload + .get("adapter") + .and_then(|adapter| adapter.get("id")) + .and_then(Value::as_str) + == Some(crate::journal_reducers::SOCKET_REPORT_ADAPTER) + }) + .count() + }; + + for _ in 0..4 { + mux.report_agent( + target_surface.id, + AgentState::Done, + AgentSource::Socket, + Some("blocked-done-session".into()), + ) + .unwrap(); + } + assert_eq!(socket_echoes(), 1, "durable Done projection must bound repeated echoes"); + + mux.agent_roster_fold_worker_running.store(false, Ordering::Release); + mux.shutdown(); + } + #[test] fn roster_fold_pages_to_a_target_beyond_the_first_journal_page() { let root = std::env::temp_dir().join(format!( From 318f4d03723b91603e30bbbde6a4f13d63bbedf6 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:35:31 -0700 Subject: [PATCH 113/133] fix(cmux-tui): use durable projection for echo coalescing --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 43 ++++++++++++------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c74570d80bde..3b8cb4ee49fc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6456,30 +6456,31 @@ impl Mux { source: AgentSource, session: Option<&str>, ) -> Option { - // Resource revisions are durable and strictly monotonic. Include the - // current revision in a transition key so an ABA sequence cannot - // reuse the key from an earlier cycle after the compatibility cache - // returns to the same state (working -> blocked -> working -> - // blocked). Repeated polls of one semantic state still return early - // through `unchanged`, so unrelated revisions do not create echo - // rows for steady-state reports. - let resource_revision = self.state.lock().unwrap().resource_revision; + // The durable projection is updated before its echo is folded. Read + // it first so a blocked roster replay still has a stable semantic + // receipt for repeated reports. + let durable_previous = self + .workspace_registry + .lock() + .unwrap() + .public_agent_projections(Some(terminal_id), None) + .ok() + .and_then(|mut projections| projections.pop()) + .map(|record| (record.state, record.source, record.source_session, record.agent)); // The compatibility cache is updated as part of the direct resource // commit, before its journal echo is folded. Prefer it for admission // so an unfinished reducer tail cannot make an ABA sequence reuse an // older idempotency key (working -> blocked -> working). - let previous = self - .agent_records - .lock() - .unwrap() - .get(terminal_id) - .map(|record| { - ( - record.state.as_str().to_string(), - record.source.as_str().to_string(), - record.session.clone(), - record.agent.clone(), - ) + let previous = durable_previous + .or_else(|| { + self.agent_records.lock().unwrap().get(terminal_id).map(|record| { + ( + record.state.as_str().to_string(), + record.source.as_str().to_string(), + record.session.clone(), + record.agent.clone(), + ) + }) }) .or_else(|| { let host = self.agent_roster.lock().unwrap(); @@ -6549,7 +6550,7 @@ impl Mux { // journal cursor would make an unrelated event defeat coalescing and // turn a steady socket poll into one durable row per event elsewhere. let material = format!( - "agent-report-echo-v3|{terminal_id}|{}|{}|{:?}|revision={resource_revision}|{previous}", + "agent-report-echo-v4|{terminal_id}|{}|{}|{:?}|{previous}", state.as_str(), source.as_str(), session, From c378cf1cec83bb775605552d6bd6b54b79595328 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:42:15 -0700 Subject: [PATCH 114/133] test(cmux-tui): allow reused hook session reopening --- .../cmux-tui-core/src/journal_reducers.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 1bc0d265475e..bd043dd1d399 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -805,6 +805,24 @@ mod tests { assert_eq!(restored.entries["term_a"].state, "idle"); } + #[test] + fn newer_hook_session_start_can_reopen_when_provider_reuses_session_id() { + let subjects = terminal_subject("term_a"); + let payload = json!({ + "adapter": {"id": "claude", "version": 1}, + "normalized": {"agent_session_id": "reused-session"} + }); + let mut roster = AgentRoster::default(); + + roster.apply(&hook_event(1, "agent.session.started", &subjects, &payload)); + roster.apply(&hook_event(2, "agent.session.ended", &subjects, &payload)); + let deltas = roster.apply(&hook_event(3, "agent.session.started", &subjects, &payload)); + + assert_eq!(deltas.len(), 1, "a newer start reopens a reused provider session id"); + assert_eq!(roster.entries["term_a"].state, "idle"); + assert!(!roster.has_ended_hook_fence("term_a")); + } + #[test] fn folds_are_idempotent_per_state_and_deterministic_across_replays() { let subjects = terminal_subject("term_a"); From e1ae84ce10aa1a11f1e21c07aa6997af272e04a0 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:42:25 -0700 Subject: [PATCH 115/133] fix(cmux-tui): reopen reused hook session ids --- cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index bd043dd1d399..9913ce71e752 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -315,7 +315,10 @@ impl AgentRoster { .unwrap_or_else(|| format!("legacy:{terminal_id}:{}", event.sequence)); if let Some(fence) = previous_fence.as_ref() { if fence.session_id == session_id { - if fence.ended || event.sequence <= fence.sequence { + // A provider may reuse its native session id. A newer + // explicit session start reopens that lifecycle, while + // all other ended-fence records remain rejected. + if (fence.ended && !is_session_start) || event.sequence <= fence.sequence { return Vec::new(); } } else if !is_session_start || !fence.ended || event.sequence <= fence.sequence { From 465c139b08cdb0e3e3f9f5598de224fe9b03f845 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:39:32 -0700 Subject: [PATCH 116/133] fix(cmux-tui): close journal before waiting for roster worker --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 3b8cb4ee49fc..7b78abf24873 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10883,12 +10883,17 @@ impl Mux { self.journal_event_changed.notify_all(); self.journal_kernel.wake_waiters(); let hook_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; - self.wait_for_agent_roster_fold_worker(hook_deadline); if !self.journal_hook_runtime.shutdown_until(hook_deadline) { eprintln!("cmux-tui: journal hook workers did not stop before the shutdown deadline"); } self.finalize_terminal_journal("shutdown"); - if let Err(error) = self.persist_agent_roster_snapshot_until(hook_deadline) { + // Stop the journal writer before waiting for the detached roster fold. + // A fold can be blocked on the registry while the writer owns its + // mutex. Waiting first leaves the session lease held after shutdown + // returns when the writer cannot make progress until finalization. + let roster_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; + self.wait_for_agent_roster_fold_worker(roster_deadline); + if let Err(error) = self.persist_agent_roster_snapshot_until(roster_deadline) { eprintln!( "cmux-tui: persisting the agent roster snapshot during shutdown failed: {error}" ); From 59d5ffd6c17c916cf91b50c07ba4a81646ed867f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:40:37 -0700 Subject: [PATCH 117/133] fix(cmux-tui): wake roster worker after journal close --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7b78abf24873..141108c24667 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10891,6 +10891,7 @@ impl Mux { // A fold can be blocked on the registry while the writer owns its // mutex. Waiting first leaves the session lease held after shutdown // returns when the writer cannot make progress until finalization. + self.journal_event_changed.notify_all(); let roster_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; self.wait_for_agent_roster_fold_worker(roster_deadline); if let Err(error) = self.persist_agent_roster_snapshot_until(roster_deadline) { From 099707f95d8cffe1812d5dbca45af0462b71e29e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:16:17 -0700 Subject: [PATCH 118/133] fix(cmux-tui): cancel roster folds during shutdown --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 141108c24667..bcce11de1c96 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5942,6 +5942,9 @@ impl Mux { // complete roster for every poll made callback cost grow with the // number of terminals. let _fold = self.agent_roster_fold.lock().unwrap(); + if self.shutting_down.load(Ordering::Acquire) { + return AgentRosterFoldProgress::WaitingForProjection; + } let (mut read_cursor, defer_record_cache) = { let host = self.agent_roster.lock().unwrap(); (host.cursor, host.startup_replay_pending) @@ -5951,6 +5954,9 @@ impl Mux { } let mut folded_records = 0usize; loop { + if self.shutting_down.load(Ordering::Acquire) { + return AgentRosterFoldProgress::WaitingForProjection; + } let page = match self .workspace_registry .lock() @@ -5989,6 +5995,9 @@ impl Mux { let mut skipped_quarantined = false; let mut reached_record_limit = false; for record in page.records { + if self.shutting_down.load(Ordering::Acquire) { + return AgentRosterFoldProgress::WaitingForProjection; + } if record.sequence <= read_cursor { continue; } @@ -6189,7 +6198,12 @@ impl Mux { } }; - match mux.fold_agent_roster_through(target_sequence) { + let fold_progress = mux.fold_agent_roster_through(target_sequence); + if mux.shutting_down.load(Ordering::Acquire) { + mux.mark_agent_roster_fold_worker_stopped(); + return; + } + match fold_progress { AgentRosterFoldProgress::MoreAvailable => { // Keep each pass bounded and yield between pages so a // large restored journal cannot monopolize a core. Wait @@ -10883,18 +10897,12 @@ impl Mux { self.journal_event_changed.notify_all(); self.journal_kernel.wake_waiters(); let hook_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; + self.wait_for_agent_roster_fold_worker(hook_deadline); if !self.journal_hook_runtime.shutdown_until(hook_deadline) { eprintln!("cmux-tui: journal hook workers did not stop before the shutdown deadline"); } self.finalize_terminal_journal("shutdown"); - // Stop the journal writer before waiting for the detached roster fold. - // A fold can be blocked on the registry while the writer owns its - // mutex. Waiting first leaves the session lease held after shutdown - // returns when the writer cannot make progress until finalization. - self.journal_event_changed.notify_all(); - let roster_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; - self.wait_for_agent_roster_fold_worker(roster_deadline); - if let Err(error) = self.persist_agent_roster_snapshot_until(roster_deadline) { + if let Err(error) = self.persist_agent_roster_snapshot_until(hook_deadline) { eprintln!( "cmux-tui: persisting the agent roster snapshot during shutdown failed: {error}" ); From e2cf3ed2f40099b3547ca7f0893be8af8c54eb75 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:23:21 -0700 Subject: [PATCH 119/133] fix(cmux-tui): wait for roster workers without fold lock --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bcce11de1c96..9e9e7fd1550b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2302,6 +2302,7 @@ pub struct Mux { /// Serialize journal folds so a later commit cannot advance the reducer /// cursor past an earlier committed record. agent_roster_fold: Mutex<()>, + agent_roster_fold_worker_wait: Mutex<()>, /// A single coalesced worker drains replay work that exceeds the bounded /// synchronous report budget. The journal remains the source of truth. agent_roster_fold_worker_running: AtomicBool, @@ -2723,6 +2724,7 @@ impl Mux { sidebar_plugin: Mutex::new(SidebarPluginRuntime::default()), agent_roster: Mutex::new(agent_roster), agent_roster_fold: Mutex::new(()), + agent_roster_fold_worker_wait: Mutex::new(()), agent_roster_fold_worker_running: AtomicBool::new(false), agent_roster_fold_worker_changed: Condvar::new(), self_reference: OnceLock::new(), @@ -6130,13 +6132,13 @@ impl Mux { } fn mark_agent_roster_fold_worker_stopped(&self) { - let _fold = self.agent_roster_fold.lock().unwrap(); + let _wait = self.agent_roster_fold_worker_wait.lock().unwrap(); self.agent_roster_fold_worker_running.store(false, Ordering::Release); self.agent_roster_fold_worker_changed.notify_all(); } fn wait_for_agent_roster_fold_worker(&self, deadline: Instant) { - let mut fold = self.agent_roster_fold.lock().unwrap(); + let mut wait = self.agent_roster_fold_worker_wait.lock().unwrap(); while self.agent_roster_fold_worker_running.load(Ordering::Acquire) { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { @@ -6146,8 +6148,8 @@ impl Mux { break; } let (next, result) = - self.agent_roster_fold_worker_changed.wait_timeout(fold, remaining).unwrap(); - fold = next; + self.agent_roster_fold_worker_changed.wait_timeout(wait, remaining).unwrap(); + wait = next; if result.timed_out() && self.agent_roster_fold_worker_running.load(Ordering::Acquire) { eprintln!( "cmux-tui: agent roster fold worker did not stop before shutdown deadline" From 2bf59afce2fb2e9c295bb28d889f39597de1a380 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:45:01 -0700 Subject: [PATCH 120/133] test(cmux-tui): fence retired terminal session starts --- .../cmux-tui-core/src/journal_reducers.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 9913ce71e752..65cf16bc3ae7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -913,7 +913,7 @@ mod tests { } #[test] - fn retired_terminal_rejects_late_events_until_a_new_session_starts() { + fn retired_terminal_rejects_late_events_even_with_a_new_session_start() { let subjects = terminal_subject("term_a"); let payload = json!({}); let mut roster = AgentRoster::default(); @@ -926,13 +926,22 @@ mod tests { assert!(roster.apply(&hook_event(2, "agent.turn.started", &subjects, &payload)).is_empty()); assert!(!roster.entries.contains_key("term_a")); - // Only a newer explicit session start clears the tombstone. + // A terminal resource id is tombstoned permanently. A later hook + // session start must not recreate roster state for that id, even when + // it claims a different provider session. assert!( roster.apply(&hook_event(1, "agent.session.started", &subjects, &payload)).is_empty() ); - let deltas = roster.apply(&hook_event(2, "agent.session.started", &subjects, &payload)); - assert_eq!(deltas.len(), 1); - assert_eq!(roster.entries["term_a"].state, "idle"); + let new_session_payload = json!({ + "normalized": {"agent_session_id": "new-session"} + }); + assert!( + roster + .apply(&hook_event(2, "agent.session.started", &subjects, &new_session_payload)) + .is_empty() + ); + assert!(!roster.entries.contains_key("term_a")); + assert!(roster.is_retired("term_a")); } #[test] From 4d5201833fd2508b18052aaf699a31fc9a0e42ee Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:45:48 -0700 Subject: [PATCH 121/133] fix(cmux-tui): keep retired terminal ids fenced --- .../crates/cmux-tui-core/src/journal_reducers.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs index 65cf16bc3ae7..7dc8d36829f9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_reducers.rs @@ -228,17 +228,11 @@ impl AgentRoster { let socket_echo = event.adapter_id() == Some(SOCKET_REPORT_ADAPTER); let detected_echo = event.adapter_id() == Some(DETECTED_REPORT_ADAPTER); let external_echo = socket_echo || detected_echo; - if let Some(retired_at) = self.retired_terminals.get(terminal_id).copied() { - // A terminal can only become live again through a newer explicit - // session start. All other delayed records remain fenced forever. - if external_echo - || event.kind != "agent.session.started" - || event.sequence <= retired_at - { - return Vec::new(); - } - self.retired_terminals.remove(terminal_id); - self.hook_fences.remove(terminal_id); + if self.retired_terminals.contains_key(terminal_id) { + // The resource registry permanently tombstones a terminal public + // id. No delayed journal row can create a new lifecycle for that + // id, including a later session-start event. + return Vec::new(); } let (state, source, session, agent, updated_at_ms) = if external_echo { // Socket echo: explicit state and timestamp carried in the From f7c0f3f5337fe0d1844c0c11cb42bf28a2c1d923 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:47:03 -0700 Subject: [PATCH 122/133] test(cmux-tui): preserve agent echo on projection read errors --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 32 +++++++++++++++++++ .../cmux-tui-core/src/workspace_registry.rs | 4 +++ .../public_projection_store.rs | 11 +++++++ 3 files changed, 47 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 9e9e7fd1550b..48bcd30b82a6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -26514,6 +26514,38 @@ mod tests { mux.shutdown(); } + #[test] + fn agent_report_echo_key_propagates_authoritative_projection_read_failure() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let terminal_id = surface.terminal_public_id().cloned().expect("workspace terminal"); + mux.agent_records.lock().unwrap().insert( + terminal_id.clone(), + TerminalAgentRecord { + state: AgentState::Working, + source: AgentSource::Socket, + session: Some("projection-read-failure".into()), + agent: None, + updated_at_ms: 1, + }, + ); + mux.workspace_registry.lock().unwrap().set_public_agent_projection_read_failure(true); + + let error = mux + .report_agent( + surface.id, + AgentState::Working, + AgentSource::Socket, + Some("projection-read-failure".into()), + ) + .unwrap_err(); + assert!( + error.to_string().contains("public agent projection read failure"), + "unexpected projection read error: {error:#}" + ); + mux.shutdown(); + } + #[test] fn agent_reports_require_a_terminal_and_never_create_a_default_projection() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index f0669938f54d..02320ccd5372 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -680,6 +680,8 @@ pub struct WorkspaceRegistry { #[cfg(test)] resource_patch_failures_remaining: Cell, #[cfg(test)] + public_agent_projection_failures_remaining: Cell, + #[cfg(test)] journal_before_commit: Option<(std::sync::mpsc::SyncSender<()>, std::sync::mpsc::Receiver<()>)>, #[cfg(test)] journal_after_commit_admission: @@ -2686,6 +2688,8 @@ impl WorkspaceRegistry { #[cfg(test)] resource_patch_failures_remaining: Cell::new(0), #[cfg(test)] + public_agent_projection_failures_remaining: Cell::new(0), + #[cfg(test)] journal_before_commit: None, #[cfg(test)] journal_after_commit_admission: None, diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs index 7cf9aff49c08..c23263e216b1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs @@ -213,6 +213,12 @@ impl WorkspaceRegistry { terminal: Option<&TerminalPublicId>, state: Option<&str>, ) -> anyhow::Result> { + #[cfg(test)] + if self.public_agent_projection_failures_remaining.get() > 0 { + self.public_agent_projection_failures_remaining + .set(self.public_agent_projection_failures_remaining.get() - 1); + anyhow::bail!("forced public agent projection read failure"); + } let mut agents = self.durable_agents(terminal, state)?; agents.retain(|agent| { !(agent.source == "hook" && agent.state == "done") @@ -231,6 +237,11 @@ impl WorkspaceRegistry { Ok(agents) } + #[cfg(test)] + pub(crate) fn set_public_agent_projection_read_failure(&self, enabled: bool) { + self.public_agent_projection_failures_remaining.set(u64::from(enabled)); + } + fn live_terminal_public_ids(&self) -> anyhow::Result> { let mut statement = self.connection.prepare( "SELECT public_id From c38ee5bc24df420c0f844215ed61558952105bb0 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:49:12 -0700 Subject: [PATCH 123/133] fix(cmux-tui): fail reports when echo authority is unreadable --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 48bcd30b82a6..6928fdc3c1cc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6471,17 +6471,16 @@ impl Mux { state: AgentState, source: AgentSource, session: Option<&str>, - ) -> Option { + ) -> anyhow::Result> { // The durable projection is updated before its echo is folded. Read // it first so a blocked roster replay still has a stable semantic // receipt for repeated reports. let durable_previous = self .workspace_registry - .lock() - .unwrap() - .public_agent_projections(Some(terminal_id), None) - .ok() - .and_then(|mut projections| projections.pop()) + .lock_until(Instant::now() + JOURNAL_DURABLE_WAIT)? + .public_agent_projections(Some(terminal_id), None)? + .into_iter() + .next() .map(|record| (record.state, record.source, record.source_session, record.agent)); // The compatibility cache is updated as part of the direct resource // commit, before its journal echo is folded. Prefer it for admission @@ -6555,7 +6554,7 @@ impl Mux { || hook_owned_external_report || socket_owned_detected_report { - return None; + return Ok(None); } let previous = previous .map(|(state, source, session, agent)| { @@ -6578,7 +6577,7 @@ impl Mux { use std::fmt::Write as _; write!(key, "{byte:02x}").expect("writing to a String cannot fail"); } - Some(key) + Ok(Some(key)) } /// Echo direct reports into the journal so a restart can rebuild the @@ -10203,7 +10202,7 @@ impl Mux { id.clone() }; let timestamp = now_ms(); - match self.agent_report_echo_key(&terminal_id, state, source, session.as_deref()) { + match self.agent_report_echo_key(&terminal_id, state, source, session.as_deref())? { Some(key) => { let ingress = self.agent_report_echo_ingress( &terminal_id, @@ -10289,7 +10288,7 @@ impl Mux { agent_state, source, source_session.as_deref(), - ) { + )? { Some(key) => { let ingress = self.agent_report_echo_ingress( terminal_id, @@ -26509,6 +26508,7 @@ mod tests { AgentSource::Socket, Some("aba-session"), ) + .unwrap() .is_none() ); mux.shutdown(); From cf63a5e2849a4718e4ff1a1a4f491848c1b88e9d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:50:21 -0700 Subject: [PATCH 124/133] fix(cmux-tui): bound durable echo authority reads --- cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs b/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs index d3a02672ff0e..1ed6659947d8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs @@ -21,7 +21,7 @@ const JOURNAL_DURABLE_BATCH_BYTES: usize = 8 * 1024 * 1024; pub(crate) const TERMINAL_OUTPUT_INGRESS_BYTES: usize = 64 * 1024; const TERMINAL_OUTPUT_BATCH_BYTES: usize = 256 * 1024; const JOURNAL_TERMINAL_FAILURE_RETRY_ATTEMPTS: usize = 6; -const JOURNAL_DURABLE_WAIT: Duration = Duration::from_secs(2); +pub(crate) const JOURNAL_DURABLE_WAIT: Duration = Duration::from_secs(2); const JOURNAL_COMMIT_RESULT_WAIT: Duration = Duration::from_secs(1); const JOURNAL_WRITER_SHUTDOWN_WAIT: Duration = Duration::from_secs(1); const JOURNAL_SQLITE_RETRY_SLICE: Duration = Duration::from_millis(100); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 6928fdc3c1cc..81b31a141fba 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6477,7 +6477,7 @@ impl Mux { // receipt for repeated reports. let durable_previous = self .workspace_registry - .lock_until(Instant::now() + JOURNAL_DURABLE_WAIT)? + .lock_until(Instant::now() + crate::journal_ingress::JOURNAL_DURABLE_WAIT)? .public_agent_projections(Some(terminal_id), None)? .into_iter() .next() From 02428a4e65d7bc5707dad139b21a7906528c8aa0 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 31 Aug 2026 02:16:35 -0700 Subject: [PATCH 125/133] test(cmux-tui): fence journal target owner lifetime --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 229bbcfabe00..a7b20a1781d6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -6859,6 +6859,34 @@ mod tests { use super::*; use crate::MuxEvent; + #[test] + fn journal_target_does_not_retain_mux_owner() { + let root = std::env::temp_dir().join(format!( + "cmux-journal-target-lifetime-{}", + crate::workspace_registry::new_uuid_v4() + )); + let mux = Mux::open_persistent("journal-target-lifetime", SurfaceOptions::default(), &root) + .unwrap(); + let mux_weak = Arc::downgrade(&mux); + let surface = + Surface::spawn_for_test(1, SurfaceOptions::default(), Arc::downgrade(&mux)).unwrap(); + let target = surface + .as_pty() + .and_then(PtySurface::journal_target) + .expect("persistent terminal has a journal target"); + + drop(mux); + assert_eq!( + mux_weak.strong_count(), + 0, + "a journal target must not keep the session owner alive" + ); + + drop(target); + drop(surface); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn agent_browser_provider_uses_a_terminal_local_daemon_session() { let mut options = SurfaceOptions { From c0e91e4625ccea54a07a25d10690384c488d3c32 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 31 Aug 2026 03:39:13 -0700 Subject: [PATCH 126/133] fix(cmux-tui): avoid retaining mux in journal reader --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 35 ++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index a7b20a1781d6..48ac34246b10 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -6107,28 +6107,35 @@ impl ChildKiller for TestChildKiller { } impl PtySurface { - fn journal_target(&self) -> Option<(Arc, Arc)> { + fn journal_target(&self) -> Option<(Weak, Arc)> { let terminal_id = self.terminal_public_id.clone()?; let mux = self.mux.upgrade()?; - (mux.terminal_journal_enabled() && self.journal_capture_supported) - .then_some((mux, terminal_id)) + if !mux.terminal_journal_enabled() || !self.journal_capture_supported { + return None; + } + // The target is carried across the blocking PTY read. Keep only the + // weak owner there, otherwise a reader that cannot be interrupted can + // keep the session lease alive after daemon shutdown. + drop(mux); + Some((self.mux.clone(), terminal_id)) } fn journal_output_if_open( &self, - (mux, terminal_id): (Arc, Arc), + (mux, terminal_id): (Weak, Arc), bytes: Vec, ) { let occurred_at_ms = crate::workspace_registry::unix_epoch_ms().unwrap_or(0); for chunk in bytes.chunks(crate::journal_ingress::TERMINAL_OUTPUT_INGRESS_BYTES) { let mut pending = chunk.to_vec(); loop { - let space_epoch = { + let Some(owner) = mux.upgrade() else { return }; + let retry = { let _gate = self.journal_capture_gate.lock().unwrap(); if !self.journal_capture_open.load(Ordering::Acquire) { return; } - let retry = match mux.try_journal_terminal_output( + match owner.try_journal_terminal_output( terminal_id.clone(), self.journal_generation.clone(), occurred_at_ms, @@ -6137,25 +6144,27 @@ impl PtySurface { Ok(retry) => retry, Err(error) => { self.journal_capture_open.store(false, Ordering::Release); - mux.request_daemon_shutdown(); + owner.request_daemon_shutdown(); eprintln!( "cmux-tui: terminal journal capture failed; stopping daemon: {error}" ); return; } - }; - let Some((retry, space_epoch)) = retry else { break }; - pending = retry; - space_epoch + } }; - if let Err(error) = mux.wait_for_terminal_journal_space(space_epoch) { + drop(owner); + let Some((retry, space_epoch)) = retry else { break }; + pending = retry; + let Some(owner) = mux.upgrade() else { return }; + if let Err(error) = owner.wait_for_terminal_journal_space(space_epoch) { self.journal_capture_open.store(false, Ordering::Release); - mux.request_daemon_shutdown(); + owner.request_daemon_shutdown(); eprintln!( "cmux-tui: terminal journal capture failed; stopping daemon: {error}" ); return; } + drop(owner); } } } From ff1cd1eee5a9059c006891039511a040863ea8c7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:46:57 -0700 Subject: [PATCH 127/133] test(cmux-tui): trace persistent mux shutdown ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 81b31a141fba..b1c63b9c825c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24874,7 +24874,16 @@ mod tests { .is_err() ); mux.shutdown(); + let mux_weak = Arc::downgrade(&mux); + eprintln!( + "cmux-tui test authority scope mux strong count after shutdown: {}", + mux_weak.strong_count() + ); drop(mux); + eprintln!( + "cmux-tui test authority scope mux strong count after drop: {}", + mux_weak.strong_count() + ); let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); let records = reopened.agent_records.lock().unwrap(); From 93e4dfb6313ffecbc2861eafd2cad09f1f0e0857 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:18:58 -0700 Subject: [PATCH 128/133] Fence PTY child reaper shutdown --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 64 ++++++++++++++++---- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 48ac34246b10..027e1a538d30 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1407,6 +1407,10 @@ pub struct PtyTerminalRuntime { /// journal barrier. reader_thread: Mutex>>, reader_completion: Arc, + /// Owned child reaper join fence. The reaper publishes native exit only + /// after PTY master close and the final exit notification are complete. + reaper_thread: Mutex>>, + reaper_completion: Arc, term: Mutex>, stream_progress: Box, mouse_encoders: Mutex>, @@ -2368,6 +2372,8 @@ impl Surface { journal_capture_active: AtomicBool::new(false), reader_thread: Mutex::new(None), reader_completion: Arc::new(ReaderCompletion::default()), + reaper_thread: Mutex::new(None), + reaper_completion: Arc::new(ReaderCompletion::default()), term: Mutex::new(Box::new(term)), stream_progress: Box::new(TerminalStreamProgress::default()), mouse_encoders: Mutex::new(Box::new(mouse_encoders)), @@ -2562,18 +2568,37 @@ impl Surface { .unwrap() = Some(reader_thread); // Child reaper: retain the native status and rendezvous with PTY EOF - // so final output is visible before the mux observes completion. - std::thread::Builder::new().name(format!("surface-{id}-wait")).spawn({ - let surface = surface.clone(); - move || { - let exit = wait_for_native_child_status(child.as_mut()); - if let Some(pty) = surface.as_pty() { - *pty.exit.lock().unwrap() = Some(exit); + // so final output is visible before the mux observes completion. Keep + // only a weak surface reference: the owned JoinHandle must not retain + // the surface and form a lifecycle cycle. + let reaper_completion = surface + .as_pty() + .expect("local PTY surface owns its reaper") + .reaper_completion + .clone(); + let reaper_surface = Arc::downgrade(&surface); + let reaper_thread = std::thread::Builder::new() + .name(format!("surface-{id}-wait")) + .spawn({ + let reaper_completion = reaper_completion.clone(); + move || { + let _reaper_completion = ReaderCompletionGuard(reaper_completion); + let exit = wait_for_native_child_status(child.as_mut()); + let Some(surface) = reaper_surface.upgrade() else { return }; + if let Some(pty) = surface.as_pty() { + *pty.exit.lock().unwrap() = Some(exit); + } + close_local_terminal_master_after_exit(&surface); + publish_local_exit_if_ready(&surface); } - close_local_terminal_master_after_exit(&surface); - publish_local_exit_if_ready(&surface); - } - })?; + })?; + surface + .as_pty() + .expect("local PTY surface owns its reaper") + .reaper_thread + .lock() + .unwrap() + .replace(reaper_thread); Ok(surface) } @@ -2840,6 +2865,8 @@ impl Surface { journal_capture_active: AtomicBool::new(false), reader_thread: Mutex::new(None), reader_completion: Arc::new(ReaderCompletion::default()), + reaper_thread: Mutex::new(None), + reaper_completion: Arc::new(ReaderCompletion::default()), term: Mutex::new(Box::new(term)), stream_progress: Box::new(TerminalStreamProgress::default()), mouse_encoders: Mutex::new(Box::new(mouse_encoders)), @@ -3883,6 +3910,8 @@ impl Surface { journal_capture_active: AtomicBool::new(false), reader_thread: Mutex::new(None), reader_completion: Arc::new(ReaderCompletion::default()), + reaper_thread: Mutex::new(None), + reaper_completion: Arc::new(ReaderCompletion::default()), term: Mutex::new(Box::new(term)), stream_progress: Box::new(TerminalStreamProgress::default()), mouse_encoders: Mutex::new(Box::new(mouse_encoders)), @@ -4108,6 +4137,8 @@ impl Surface { journal_capture_active: AtomicBool::new(false), reader_thread: Mutex::new(None), reader_completion: Arc::new(ReaderCompletion::default()), + reaper_thread: Mutex::new(None), + reaper_completion: Arc::new(ReaderCompletion::default()), term: Mutex::new(Box::new(term)), stream_progress: Box::new(TerminalStreamProgress::default()), mouse_encoders: Mutex::new(Box::new(mouse_encoders)), @@ -4192,6 +4223,17 @@ impl Surface { ); } } + if pty.reaper_completion.wait_until(deadline) { + if let Some(reaper) = pty.reaper_thread.lock().unwrap().take() { + if reaper.join().is_err() { + eprintln!("cmux-tui: terminal child reaper thread panicked during shutdown"); + } + } + } else { + eprintln!( + "cmux-tui: terminal child reaper did not stop before the shared shutdown deadline" + ); + } // A reader that is blocked in the PTY has an even capture epoch and // does not delay shutdown. Close the gate between updates. If one // update crossed the reader deadline, wait through the active-update From 90484df05010b2a90fe9fb6a4c2fa37b50624e8a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:19:34 -0700 Subject: [PATCH 129/133] fix(cmux-tui): fence roster fold worker completion --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 124 ++++++++++++++++------- 1 file changed, 86 insertions(+), 38 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b1c63b9c825c..dedc370c2e07 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2171,6 +2171,17 @@ struct ConfigReloadState { applied: u64, } +struct JournalEventSignal { + epoch: Mutex, + changed: Condvar, +} + +impl Default for JournalEventSignal { + fn default() -> Self { + Self { epoch: Mutex::new(0), changed: Condvar::new() } + } +} + /// Describes why an owner did not confirm a requested configuration reload. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ConfigReloadError { @@ -2303,6 +2314,7 @@ pub struct Mux { /// cursor past an earlier committed record. agent_roster_fold: Mutex<()>, agent_roster_fold_worker_wait: Mutex<()>, + agent_roster_fold_worker_handle: Mutex>>, /// A single coalesced worker drains replay work that exceeds the bounded /// synchronous report budget. The journal remains the source of truth. agent_roster_fold_worker_running: AtomicBool, @@ -2328,8 +2340,7 @@ pub struct Mux { journal_hook_runtime: Arc, /// Wake-only signal for durable journal subscribers. Consumers always /// reread SQLite by cursor, so missed or coalesced notifications are safe. - journal_event_epoch: Mutex, - journal_event_changed: Condvar, + journal_event_signal: Arc, /// True once a skipped terminal-host reconnect checkpoint has been logged. /// A machine resume reconnects every hosted terminal at once, so /// per-terminal reporting would emit N warnings for one underlying @@ -2725,6 +2736,7 @@ impl Mux { agent_roster: Mutex::new(agent_roster), agent_roster_fold: Mutex::new(()), agent_roster_fold_worker_wait: Mutex::new(()), + agent_roster_fold_worker_handle: Mutex::new(None), agent_roster_fold_worker_running: AtomicBool::new(false), agent_roster_fold_worker_changed: Condvar::new(), self_reference: OnceLock::new(), @@ -2738,8 +2750,7 @@ impl Mux { journal_ingress, journal_hook_dispatcher_started: AtomicBool::new(false), journal_hook_runtime: Arc::new(crate::journal_hooks::JournalHookRuntime::default()), - journal_event_epoch: Mutex::new(0), - journal_event_changed: Condvar::new(), + journal_event_signal: Arc::new(JournalEventSignal::default()), reconnect_checkpoint_skip_reported: AtomicBool::new(false), diagnostic_reporter: OnceLock::new(), pending_diagnostic: Mutex::new(None), @@ -5486,9 +5497,9 @@ impl Mux { } fn wake_journal_event_waiters(&self) { - let mut epoch = self.journal_event_epoch.lock().unwrap(); + let mut epoch = self.journal_event_signal.epoch.lock().unwrap(); *epoch = epoch.wrapping_add(1); - self.journal_event_changed.notify_all(); + self.journal_event_signal.changed.notify_all(); } #[cfg_attr(not(test), allow(dead_code))] @@ -5687,15 +5698,23 @@ impl Mux { } pub(crate) fn journal_event_epoch(&self) -> u64 { - *self.journal_event_epoch.lock().unwrap() + *self.journal_event_signal.epoch.lock().unwrap() } pub(crate) fn wait_for_journal_event(&self, epoch: u64, timeout: Duration) -> u64 { - let current = self.journal_event_epoch.lock().unwrap(); + Self::wait_for_journal_event_signal(&self.journal_event_signal, epoch, timeout) + } + + fn wait_for_journal_event_signal( + signal: &JournalEventSignal, + epoch: u64, + timeout: Duration, + ) -> u64 { + let current = signal.epoch.lock().unwrap(); if *current != epoch { return *current; } - let (current, _) = self.journal_event_changed.wait_timeout(current, timeout).unwrap(); + let (current, _) = signal.changed.wait_timeout(current, timeout).unwrap(); *current } @@ -6106,9 +6125,9 @@ impl Mux { } } - /// Start one detached reducer worker when a bounded synchronous fold did - /// not reach its target. The worker owns no durable state and is woken by - /// the journal epoch; every pass rereads SQLite from the reducer cursor. + /// Start one coalesced reducer worker when a bounded synchronous fold did + /// not reach its target. The join handle is retained until completion so + /// shutdown can fence the worker before releasing the session lease. fn schedule_agent_roster_fold_worker(&self) { if self .agent_roster_fold_worker_running @@ -6118,16 +6137,38 @@ impl Mux { return; } + if let Some(handle) = self.agent_roster_fold_worker_handle.lock().unwrap().take() { + let _ = handle.join(); + } + let Some(mux) = self.self_reference.get().cloned() else { self.mark_agent_roster_fold_worker_stopped(); return; }; - if let Err(error) = std::thread::Builder::new() + let signal = Arc::clone(&self.journal_event_signal); + let worker = std::thread::Builder::new() .name("agent-roster-fold".into()) - .spawn(move || Self::run_agent_roster_fold_worker(mux)) - { - self.mark_agent_roster_fold_worker_stopped(); - eprintln!("cmux-tui: could not start agent roster fold worker: {error}"); + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::run_agent_roster_fold_worker(mux.clone(), signal) + })); + if result.is_err() { + if let Some(mux) = mux.upgrade() { + mux.report_internal_diagnostic("agent roster fold worker panicked"); + } + } + if let Some(mux) = mux.upgrade() { + mux.mark_agent_roster_fold_worker_stopped(); + } + }); + match worker { + Ok(handle) => { + *self.agent_roster_fold_worker_handle.lock().unwrap() = Some(handle); + } + Err(error) => { + self.mark_agent_roster_fold_worker_stopped(); + eprintln!("cmux-tui: could not start agent roster fold worker: {error}"); + } } } @@ -6157,9 +6198,17 @@ impl Mux { break; } } + drop(wait); + if Instant::now() < deadline + && !self.agent_roster_fold_worker_running.load(Ordering::Acquire) + { + if let Some(handle) = self.agent_roster_fold_worker_handle.lock().unwrap().take() { + let _ = handle.join(); + } + } } - fn run_agent_roster_fold_worker(mux: Weak) { + fn run_agent_roster_fold_worker(mux: Weak, signal: Arc) { let mut projection_retry_attempts = 0u32; let mut journal_retry_attempts = 0u32; loop { @@ -6192,10 +6241,9 @@ impl Mux { return; } let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event( - epoch, - agent_roster_projection_retry_delay(journal_retry_attempts), - ); + let timeout = agent_roster_projection_retry_delay(journal_retry_attempts); + drop(mux); + Self::wait_for_journal_event_signal(&signal, epoch, timeout); continue; } }; @@ -6214,7 +6262,12 @@ impl Mux { // at a bounded cadence. projection_retry_attempts = 0; let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_BATCH_WAIT); + drop(mux); + Self::wait_for_journal_event_signal( + &signal, + epoch, + AGENT_ROSTER_FOLD_WORKER_BATCH_WAIT, + ); } AgentRosterFoldProgress::WaitingForProjection => { // Retry the durable projection receipt. Each failed @@ -6253,10 +6306,9 @@ impl Mux { // handles repeated failures while the epoch handles // normal commits and successful cleanup wakes promptly. let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event( - epoch, - agent_roster_projection_retry_delay(projection_retry_attempts), - ); + let timeout = agent_roster_projection_retry_delay(projection_retry_attempts); + drop(mux); + Self::wait_for_journal_event_signal(&signal, epoch, timeout); } AgentRosterFoldProgress::ReachedTarget => { projection_retry_attempts = 0; @@ -6277,7 +6329,12 @@ impl Mux { "cmux-tui: rechecking the agent roster journal head failed: {error}" ); let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event(epoch, AGENT_ROSTER_FOLD_WORKER_WAIT); + drop(mux); + Self::wait_for_journal_event_signal( + &signal, + epoch, + AGENT_ROSTER_FOLD_WORKER_WAIT, + ); continue; } }; @@ -10895,7 +10952,7 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); self.config_reload_changed.notify_all(); - self.journal_event_changed.notify_all(); + self.wake_journal_event_waiters(); self.journal_kernel.wake_waiters(); let hook_deadline = Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT; self.wait_for_agent_roster_fold_worker(hook_deadline); @@ -24874,16 +24931,7 @@ mod tests { .is_err() ); mux.shutdown(); - let mux_weak = Arc::downgrade(&mux); - eprintln!( - "cmux-tui test authority scope mux strong count after shutdown: {}", - mux_weak.strong_count() - ); drop(mux); - eprintln!( - "cmux-tui test authority scope mux strong count after drop: {}", - mux_weak.strong_count() - ); let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root).unwrap(); let records = reopened.agent_records.lock().unwrap(); From e7346b41db4232b915e3d5947afb1490309ada91 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:20:11 -0700 Subject: [PATCH 130/133] fix(cmux-tui): release journal writer mux before waits --- .../crates/cmux-tui-core/src/journal_ingress.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs b/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs index 1ed6659947d8..5a2939572a64 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs @@ -1010,8 +1010,14 @@ fn run(mux: Weak, receivers: JournalIngressReceivers) { return; } if retryable_sqlite_error(&error) { - let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event(epoch, delay.min(remaining)); + let journal = mux.shared_journal_handle(); + let epoch = journal.epoch(); + drop(mux); + // Keep the retry wake independent from the owning + // mux so a blocked writer cannot keep shutdown + // alive. The journal kernel receives the same + // commit and shutdown notifications. + journal.wait(epoch, delay.min(remaining)); delay = (delay * 2).min(Duration::from_secs(1)); continue; } @@ -1028,8 +1034,10 @@ fn run(mux: Weak, receivers: JournalIngressReceivers) { if receivers.state.pause_nonretryable_failure_for_test() { continue; } - let epoch = mux.journal_event_epoch(); - mux.wait_for_journal_event(epoch, delay); + let journal = mux.shared_journal_handle(); + let epoch = journal.epoch(); + drop(mux); + journal.wait(epoch, delay); delay = (delay * 2).min(Duration::from_secs(1)); continue; } else if batch[0].completion.is_none() { From b7a5becd39c4c986842f9bfd4147ecb3abc077d9 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:23:28 -0700 Subject: [PATCH 131/133] style(cmux-tui): format lifecycle fence --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 5 ++--- cmux-tui/crates/cmux-tui-core/src/surface.rs | 12 ++++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index dedc370c2e07..21d4538fa92e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6146,9 +6146,8 @@ impl Mux { return; }; let signal = Arc::clone(&self.journal_event_signal); - let worker = std::thread::Builder::new() - .name("agent-roster-fold".into()) - .spawn(move || { + let worker = + std::thread::Builder::new().name("agent-roster-fold".into()).spawn(move || { let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { Self::run_agent_roster_fold_worker(mux.clone(), signal) })); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 027e1a538d30..ce6c08fee8e8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2571,15 +2571,11 @@ impl Surface { // so final output is visible before the mux observes completion. Keep // only a weak surface reference: the owned JoinHandle must not retain // the surface and form a lifecycle cycle. - let reaper_completion = surface - .as_pty() - .expect("local PTY surface owns its reaper") - .reaper_completion - .clone(); + let reaper_completion = + surface.as_pty().expect("local PTY surface owns its reaper").reaper_completion.clone(); let reaper_surface = Arc::downgrade(&surface); - let reaper_thread = std::thread::Builder::new() - .name(format!("surface-{id}-wait")) - .spawn({ + let reaper_thread = + std::thread::Builder::new().name(format!("surface-{id}-wait")).spawn({ let reaper_completion = reaper_completion.clone(); move || { let _reaper_completion = ReaderCompletionGuard(reaper_completion); From 9d65f8832489dff3868b37e5f05c8b8b40ca6e4b Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:31:47 -0700 Subject: [PATCH 132/133] fix(cmux-tui): release roster registry guards before waits --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 21d4538fa92e..29822a078643 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6217,12 +6217,11 @@ impl Mux { return; } - let target_sequence = match mux - .workspace_registry - .lock() - .unwrap() - .session_journal_after(0, 1) - { + let journal_head = { + let registry = mux.workspace_registry.lock().unwrap(); + registry.session_journal_after(0, 1) + }; + let target_sequence = match journal_head { Ok(page) => { journal_retry_attempts = 0; page.head_sequence @@ -6316,12 +6315,11 @@ impl Mux { // cleared is either visible in the head or changes the // epoch and causes this worker to retain ownership. let observed_epoch = mux.journal_event_epoch(); - let current_head = match mux - .workspace_registry - .lock() - .unwrap() - .session_journal_after(0, 1) - { + let journal_head = { + let registry = mux.workspace_registry.lock().unwrap(); + registry.session_journal_after(0, 1) + }; + let current_head = match journal_head { Ok(page) => page.head_sequence, Err(error) => { eprintln!( From 39fbc231b06e38ff0e51e2acb28542523546940f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:40:24 -0700 Subject: [PATCH 133/133] fix(cmux-tui): join roster worker during mux drop --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 29822a078643..0598b6da880a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -17998,6 +17998,17 @@ fn sidebar_retry_delay(failures: u32) -> Duration { impl Drop for Mux { fn drop(&mut self) { + // A last-owner drop cannot race an upgraded worker Arc, but the + // worker may still be asleep between Weak upgrades. Wake it and join + // the retained handle before the signal and journal fields drop. + self.shutting_down.store(true, Ordering::Release); + self.wake_journal_event_waiters(); + self.journal_kernel.wake_waiters(); + if let Ok(handle) = self.agent_roster_fold_worker_handle.get_mut() + && let Some(handle) = handle.take() + { + let _ = handle.join(); + } self.finalize_terminal_journal("mux drop"); self.journal_kernel.shutdown(); if let Ok(runtime) = self.browser_runtime.get_mut()