From 2472251831862ab061967c8a2d1cb7f59b7f015d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:14:41 +0000 Subject: [PATCH 01/42] test(cmux-tui-core): the person's key waits for no shared lock (R-DAEMON-STATE S1 red) A key line from an attached view goes through the real connection path (the line handler, the connection's dispatcher, the send handler) while another thread holds one shared lock for 300 ms. #31940 took Mux.state off the key; the key still waits for the client registry lock (five takes per line), the link peers lock, the host proxy authors lock, the surface admission lock, the creating-input lock, the client sizing lock and the activity lock. Their other holders are every connect, attach, resize and request of every client. Mux.state and the workspace registry rows are controls (pass on the base). Two keys go in a row, so a lock taken after the PTY write (sizing, activity) shows as a delay of the second key on the connection's dispatcher. Test-only hold hooks (cfg(test)) in creating_input.rs, terminal_sizing.rs, activity.rs and agent_session_attach.rs. No product change. Design: handoff/rewrites/R-DAEMON-STATE/DESIGN.md section 2.6 (stage S1). --- .../cmux-tui-core/src/mux/creating_input.rs | 9 + .../cmux-tui-core/src/mux/terminal_sizing.rs | 8 + .../cmux-tui-core/src/server/activity.rs | 9 + .../src/server/agent_session_attach.rs | 8 + .../crates/cmux-tui-core/src/server/tests.rs | 1 + .../src/server/tests/input_shared_locks.rs | 221 ++++++++++++++++++ 6 files changed, 256 insertions(+) create mode 100644 cmux-tui/crates/cmux-tui-core/src/server/tests/input_shared_locks.rs diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs index 3ae76d43f380..9266df1e1b32 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs @@ -93,6 +93,15 @@ fn env_millis(name: &str) -> Option { } impl Mux { + /// Runs `hold` while this thread holds the creating-input lock (test hook: a person's key must not wait for + /// it, R-DAEMON-STATE S1). + #[cfg(test)] + pub(crate) fn hold_creating_input_lock_for_test(&self, hold: impl FnOnce()) { + let _terminals = + self.creating_inputs.terminals.lock().unwrap_or_else(PoisonError::into_inner); + hold(); + } + /// A creation that names `terminal_id` was received: input for the id is queued from now on. /// Returns the generation the creation must end, or `None` when it does not own the id's /// queue: a live terminal already has the id (a refused or replayed create: input goes to diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs index 1f771b53d6d2..a0508cdf0763 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs @@ -8,6 +8,14 @@ pub(super) use grid_hold::AppliedSeat; pub(crate) mod host_policy; impl Mux { + /// Runs `hold` while this thread holds the client sizing lock (test hook: a person's key must not wait for it, + /// R-DAEMON-STATE S1). + #[cfg(test)] + pub(crate) fn hold_client_sizing_lock_for_test(&self, hold: impl FnOnce()) { + let _sizing = self.client_sizing.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + hold(); + } + /// Activity by one client view: attach, explicit focus-click, or keyboard, /// paste or mouse input. Under the default `latest` policy the view takes /// the grid when it has reported a viewport. Returns whether the diff --git a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs index 2a478abbb1d4..a5a988877c5b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs @@ -45,6 +45,15 @@ fn now_ms() -> u64 { } impl ActivityStream { + /// Runs `hold` while this thread holds the activity lock (test hook: a person's key must not wait for it, + /// R-DAEMON-STATE S1). + #[cfg(test)] + pub(crate) fn hold_lock_for_test(&self, hold: impl FnOnce()) { + let (lock, _) = &*self.shared; + let _state = lock.lock().unwrap_or_else(PoisonError::into_inner); + hold(); + } + /// Input from a person's attached client (callers filter unattached one-shot sends). pub(crate) fn note_user_input(&self) { self.update(|state| state.user_input_ms = Some(now_ms())); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs index 96932c47d39a..3c4e51cfb8bc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs @@ -141,6 +141,14 @@ impl AttachState { } impl AgentSessions { + /// Runs `hold` while this thread holds the host proxy authors lock (test hook: a person's key must not wait for + /// it, R-DAEMON-STATE S1). + #[cfg(test)] + pub(super) fn hold_proxy_authors_for_test(&self, hold: impl FnOnce()) { + let _authors = self.proxy_authors.lock().unwrap_or_else(|e| e.into_inner()); + hold(); + } + /// The live attachment of `surface` on `client`; an ended one is dropped. fn attachment(&self, client: u64, surface: SurfaceId) -> Option> { let mut state = self.lock(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests.rs index 5b90c03286c7..cfbf932edb81 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests.rs @@ -540,6 +540,7 @@ mod identify_before_queue; mod incognito_orphans; mod incognito_remote; mod input_lane; +mod input_shared_locks; mod layout_receipts_shutdown; mod outbound_and_scheduler; mod read_barrier; diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/input_shared_locks.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/input_shared_locks.rs new file mode 100644 index 000000000000..a3a52e6e65ab --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/input_shared_locks.rs @@ -0,0 +1,221 @@ +//! R-DAEMON-STATE stage S1 (handoff/rewrites/R-DAEMON-STATE/DESIGN.md, section 2.6): the person's key takes no lock +//! that another terminal, another connection, a reader or a writer of the session takes. +//! +//! #31940 took the session lock (`Mux::state`) off the key. The key still waits for other shared locks: the client +//! registry (five times per line), the link peers, the host proxy authors, the surface admission budget, the client +//! sizing lock, the creating-input lock and the activity lock. Their other holders are every connect, attach, +//! resize and request of every client, and the sizing and creating-input holders take `Mux::state` and resize PTYs +//! under them. Under the L6 storm the fixed build still showed a daemon key part of p99 20-24 ms and max 74-223 ms +//! (STATUS 7761). +//! +//! The test drives a key line through the real connection path (the line handler, the connection's dispatcher, the +//! `send` handler) while another thread holds one shared lock for 300 ms. A key that waits for the lock reaches its +//! PTY after about 300 ms; a key that does not wait reaches it in well under a millisecond. Two keys go in a row: a +//! lock taken after the PTY write (sizing, activity) delays the second key, which waits on the connection's +//! dispatcher behind the first. + +// The test terminal's in-process PTY runtime and the host proxy lock exist on Unix only (as for input_lane.rs). +#![cfg(unix)] + +use super::*; +use base64::Engine; +use std::sync::mpsc; +use std::time::Instant; + +/// How long the other thread holds each lock. +const HOLD: Duration = Duration::from_millis(300); +/// A key that took longer than this waited for the held lock. Well above scheduling noise on a loaded fleet host, +/// well below `HOLD`. +const BUDGET: Duration = Duration::from_millis(150); + +/// Records when each write reached the test terminal's PTY input. +struct TimedInput(mpsc::Sender<(Instant, Vec)>); + +impl Write for TimedInput { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let _ = self.0.send((Instant::now(), bytes.to_vec())); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// One attached typing connection, as the app's terminal view makes it: its own client, its own dispatcher, a view +/// that joined the terminal's sizing. +struct TypingConnection { + mux: Arc, + client: u64, + writer: MessageWriter, + outbound: Arc, + scheduler: Arc, + surface: SurfaceId, + input: mpsc::Receiver<(Instant, Vec)>, +} + +impl TypingConnection { + fn open() -> Self { + let mux = test_mux(); + let terminal = mux.new_workspace(Some("typing".into()), Some((80, 24))).unwrap(); + let (sent, input) = mpsc::channel(); + terminal.replace_input_writer_for_test(Box::new(TimedInput(sent))); + let (writer, outbound) = captured_writer(); + let client = mux.control_clients.register(ClientTransport::Unix, writer.clone()); + attach_test_view(&mux, client, terminal.id, &writer); + let scheduler = + Arc::new(ConnectionSurfaceScheduler::new(mux.surface_operation_admission.clone())); + Self { mux, client, writer, outbound, scheduler, surface: terminal.id, input } + } + + /// Hands one `send` line to the connection, as its reader thread does. + fn submit(&self, id: &str, byte: u8) { + let line = json!({ + "id": id, + "cmd": "send", + "surface": self.surface, + "bytes": base64::engine::general_purpose::STANDARD.encode([byte]), + }) + .to_string(); + assert!( + handle_connection_frame( + &self.mux, + self.client, + ClientTransport::Unix, + &line, + &self.writer, + &self.scheduler, + ), + "the connection closed on a key line" + ); + } + + /// When `byte` reached the PTY, or `None` after 5 s. + fn arrival(&self, byte: u8) -> Option { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + let (at, bytes) = self.input.recv_timeout(remaining).ok()?; + if bytes.contains(&byte) { + return Some(at); + } + } + } + + /// Waits for the replies to `ids`, so the connection's dispatcher is idle again. + fn settle(&self, ids: &[&str]) { + let deadline = Instant::now() + Duration::from_secs(5); + let mut left: Vec<&str> = ids.to_vec(); + while !left.is_empty() && Instant::now() < deadline { + match self.outbound.try_pop() { + Some(line) => { + let value: Value = serde_json::from_str(&line).unwrap_or_default(); + if let Some(id) = value["id"].as_str() { + if value["type"] == "response" || value.get("ok").is_some() { + assert_eq!(value["ok"], true, "key {id} failed: {value}"); + } + left.retain(|pending| *pending != id); + } + } + None => std::thread::sleep(Duration::from_millis(1)), + } + } + assert!(left.is_empty(), "no reply within 5 s to {left:?}"); + } + + fn close(self) { + self.scheduler.finish_and_wait(); + disconnect_client(&self.mux, self.client, false); + self.mux.shutdown(); + } +} + +/// Holds one shared lock on another thread until released or `HOLD` passed. +type Holder = fn(&Arc, &dyn Fn()); + +/// Every shared lock the key line may meet, each with a holder. `Mux.state` and the workspace registry are controls: +/// #31940 already took them off the key, so their rows pass on the base too. +fn hold_each() -> Vec<(&'static str, Holder)> { + let mut locks: Vec<(&'static str, Holder)> = Vec::new(); + locks.push(("client registry (ClientRegistry.state)", |mux, hold| { + let _held = mux.control_clients.state.lock().unwrap_or_else(|e| e.into_inner()); + hold(); + })); + locks.push(("link peers (RemoteRelayState.peers)", |mux, hold| { + let _held = mux.remote_relay().peers.lock().unwrap_or_else(|e| e.into_inner()); + hold(); + })); + locks.push(("host proxy authors (AgentSessions.proxy_authors)", |mux, hold| { + mux.control_clients.agent_sessions.hold_proxy_authors_for_test(hold); + })); + locks.push(("surface admission (ServerSurfaceOperationAdmission.state)", |mux, hold| { + let _held = mux.surface_operation_admission.state.lock().unwrap_or_else(|e| e.into_inner()); + hold(); + })); + locks.push(("creating input (CreatingInputs.terminals)", |mux, hold| { + mux.hold_creating_input_lock_for_test(hold); + })); + locks.push(("client sizing (Mux.client_sizing)", |mux, hold| { + mux.hold_client_sizing_lock_for_test(hold); + })); + locks.push(("activity (ActivityStream)", |mux, hold| mux.activity.hold_lock_for_test(hold))); + locks.push(("session (Mux.state)", |mux, hold| mux.with_state(|_| hold()))); + locks.push(("workspace registry (Mux.workspace_registry)", |mux, hold| { + let _held = mux.workspace_registry.lock().unwrap_or_else(|e| e.into_inner()); + hold(); + })); + locks +} + +#[test] +fn a_key_waits_for_no_shared_lock() { + let connection = TypingConnection::open(); + // The first key of a view may take the long way; every key after it must not wait. + connection.submit("warm", b'w'); + assert!(connection.arrival(b'w').is_some(), "the first key never reached its terminal"); + connection.settle(&["warm"]); + + let mut waits = Vec::new(); + for (index, (name, holder)) in hold_each().into_iter().enumerate() { + let first = b'a' + u8::try_from(index * 2).unwrap_or(0); + let second = first + 1; + let (held, holding) = mpsc::channel(); + let (release, released) = mpsc::channel::<()>(); + let mux = connection.mux.clone(); + let other = std::thread::spawn(move || { + holder(&mux, &|| { + let _ = held.send(()); + let _ = released.recv_timeout(HOLD); + }); + }); + holding.recv().expect("the other thread took its lock"); + let started = Instant::now(); + let (first_id, second_id) = (format!("{index}-first"), format!("{index}-second")); + connection.submit(&first_id, first); + connection.submit(&second_id, second); + let first_at = connection.arrival(first); + let second_at = connection.arrival(second); + let _ = release.send(()); + other.join().expect("the lock holder ended"); + connection.settle(&[first_id.as_str(), second_id.as_str()]); + let waited = match (first_at, second_at) { + (Some(first_at), Some(second_at)) => first_at.max(second_at) - started, + _ => Duration::MAX, + }; + eprintln!("SHARED-LOCKS key wait while {name} is held {HOLD:?}: {waited:?}"); + waits.push((name, waited)); + } + connection.close(); + + let blocked: Vec = waits + .iter() + .filter(|(_, waited)| *waited >= BUDGET) + .map(|(name, waited)| format!("{name}: {waited:?}")) + .collect(); + assert!( + blocked.is_empty(), + "the person's keys waited for {} shared lock(s) held by another thread (budget {BUDGET:?}, held \ + {HOLD:?}): {}", + blocked.len(), + blocked.join("; ") + ); +} From 7dc4598d2212bae8d4f802826aeec4efa5d98332 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:05:48 +0000 Subject: [PATCH 02/42] test(cmux-tui-core): a key waits for no client registry lock (R-DAEMON-STATE S1 step 1 red) The previous commit's census test held nine shared locks in turn; this commit splits it by fix step. Step 1 is the client registry lock (ClientRegistry.state), which a key line takes about nine times (handoff checks, remote-client and remote-device checks, the request's actor, the kept terminal, the reply's incognito filter) while every connect, disconnect, attach and detach of every client holds it. The test turns on the stage switch CMUX_TUI_RDS_INPUT (rds_switch.rs, inert in this commit) and drives two keys through the real connection path while another thread holds the lock for 300 ms. A second test keeps Mux.state and the workspace registry off the key on the new path (#31940). The hold hooks of the other locks leave with this commit; they come back with their fix steps. --- cmux-tui/crates/cmux-tui-core/src/lib.rs | 2 + .../cmux-tui-core/src/mux/creating_input.rs | 9 -- .../cmux-tui-core/src/mux/terminal_sizing.rs | 8 -- .../crates/cmux-tui-core/src/rds_switch.rs | 35 ++++++ .../cmux-tui-core/src/server/activity.rs | 9 -- .../src/server/agent_session_attach.rs | 8 -- .../crates/cmux-tui-core/src/server/tests.rs | 2 +- ...nput_shared_locks.rs => rds_input_lane.rs} | 111 +++++++++--------- 8 files changed, 92 insertions(+), 92 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/rds_switch.rs rename cmux-tui/crates/cmux-tui-core/src/server/tests/{input_shared_locks.rs => rds_input_lane.rs} (63%) diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index 51cf956d30d0..3ea4dd108646 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -73,6 +73,8 @@ mod pty_write; mod terminal_title; use cmux_tui_remote_access::remote_relay_state; mod login_shell; +/// The R-DAEMON-STATE stage switches (one per stage, deleted when the stage passes). +mod rds_switch; pub mod release; mod request_origin; pub mod resource; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs index 9266df1e1b32..3ae76d43f380 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs @@ -93,15 +93,6 @@ fn env_millis(name: &str) -> Option { } impl Mux { - /// Runs `hold` while this thread holds the creating-input lock (test hook: a person's key must not wait for - /// it, R-DAEMON-STATE S1). - #[cfg(test)] - pub(crate) fn hold_creating_input_lock_for_test(&self, hold: impl FnOnce()) { - let _terminals = - self.creating_inputs.terminals.lock().unwrap_or_else(PoisonError::into_inner); - hold(); - } - /// A creation that names `terminal_id` was received: input for the id is queued from now on. /// Returns the generation the creation must end, or `None` when it does not own the id's /// queue: a live terminal already has the id (a refused or replayed create: input goes to diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs index a0508cdf0763..1f771b53d6d2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs @@ -8,14 +8,6 @@ pub(super) use grid_hold::AppliedSeat; pub(crate) mod host_policy; impl Mux { - /// Runs `hold` while this thread holds the client sizing lock (test hook: a person's key must not wait for it, - /// R-DAEMON-STATE S1). - #[cfg(test)] - pub(crate) fn hold_client_sizing_lock_for_test(&self, hold: impl FnOnce()) { - let _sizing = self.client_sizing.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - hold(); - } - /// Activity by one client view: attach, explicit focus-click, or keyboard, /// paste or mouse input. Under the default `latest` policy the view takes /// the grid when it has reported a viewport. Returns whether the diff --git a/cmux-tui/crates/cmux-tui-core/src/rds_switch.rs b/cmux-tui/crates/cmux-tui-core/src/rds_switch.rs new file mode 100644 index 000000000000..8c7a1a934c5e --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/rds_switch.rs @@ -0,0 +1,35 @@ +//! The runtime switches of the R-DAEMON-STATE rewrite (handoff/rewrites/R-DAEMON-STATE/DESIGN.md, +//! section 3): one per stage, read once from the daemon's environment, deleted together with the +//! old path when the stage passes. A switch only chooses between the stage's new path and the old +//! one; both stay correct, so an A/B can run either arm in the same build. + +use std::sync::atomic::{AtomicU8, Ordering}; + +const UNREAD: u8 = 0; +const OFF: u8 = 1; +const ON: u8 = 2; + +/// Stage S1, the input lane: a connection's lines and the person's keys read the facts of the +/// session's connections from a published snapshot, never through a shared lock. +/// `CMUX_TUI_RDS_INPUT=1` turns it on. +static INPUT_LANE: AtomicU8 = AtomicU8::new(UNREAD); + +/// Whether stage S1 (the input lane) is on. The environment is read once; a race of two first +/// reads only reads it twice. +pub(crate) fn input_lane() -> bool { + match INPUT_LANE.load(Ordering::Relaxed) { + ON => true, + OFF => false, + _ => { + let on = std::env::var("CMUX_TUI_RDS_INPUT").is_ok_and(|value| value == "1"); + INPUT_LANE.store(if on { ON } else { OFF }, Ordering::Relaxed); + on + } + } +} + +/// Turns stage S1 on or off for this test process (nextest runs each test in its own process). +#[cfg(test)] +pub(crate) fn set_input_lane_for_test(on: bool) { + INPUT_LANE.store(if on { ON } else { OFF }, Ordering::Relaxed); +} diff --git a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs index a5a988877c5b..2a478abbb1d4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs @@ -45,15 +45,6 @@ fn now_ms() -> u64 { } impl ActivityStream { - /// Runs `hold` while this thread holds the activity lock (test hook: a person's key must not wait for it, - /// R-DAEMON-STATE S1). - #[cfg(test)] - pub(crate) fn hold_lock_for_test(&self, hold: impl FnOnce()) { - let (lock, _) = &*self.shared; - let _state = lock.lock().unwrap_or_else(PoisonError::into_inner); - hold(); - } - /// Input from a person's attached client (callers filter unattached one-shot sends). pub(crate) fn note_user_input(&self) { self.update(|state| state.user_input_ms = Some(now_ms())); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs index 3c4e51cfb8bc..96932c47d39a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs @@ -141,14 +141,6 @@ impl AttachState { } impl AgentSessions { - /// Runs `hold` while this thread holds the host proxy authors lock (test hook: a person's key must not wait for - /// it, R-DAEMON-STATE S1). - #[cfg(test)] - pub(super) fn hold_proxy_authors_for_test(&self, hold: impl FnOnce()) { - let _authors = self.proxy_authors.lock().unwrap_or_else(|e| e.into_inner()); - hold(); - } - /// The live attachment of `surface` on `client`; an ended one is dropped. fn attachment(&self, client: u64, surface: SurfaceId) -> Option> { let mut state = self.lock(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests.rs index cfbf932edb81..7eb5706607b4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests.rs @@ -540,9 +540,9 @@ mod identify_before_queue; mod incognito_orphans; mod incognito_remote; mod input_lane; -mod input_shared_locks; mod layout_receipts_shutdown; mod outbound_and_scheduler; +mod rds_input_lane; mod read_barrier; mod resource_attach; mod resource_waits; diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/input_shared_locks.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs similarity index 63% rename from cmux-tui/crates/cmux-tui-core/src/server/tests/input_shared_locks.rs rename to cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index a3a52e6e65ab..095cdc3c8f66 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/input_shared_locks.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -1,20 +1,18 @@ //! R-DAEMON-STATE stage S1 (handoff/rewrites/R-DAEMON-STATE/DESIGN.md, section 2.6): the person's key takes no lock //! that another terminal, another connection, a reader or a writer of the session takes. //! -//! #31940 took the session lock (`Mux::state`) off the key. The key still waits for other shared locks: the client -//! registry (five times per line), the link peers, the host proxy authors, the surface admission budget, the client -//! sizing lock, the creating-input lock and the activity lock. Their other holders are every connect, attach, -//! resize and request of every client, and the sizing and creating-input holders take `Mux::state` and resize PTYs -//! under them. Under the L6 storm the fixed build still showed a daemon key part of p99 20-24 ms and max 74-223 ms -//! (STATUS 7761). +//! #31940 took the session lock (`Mux::state`) off the key. The key line still takes the client registry lock about +//! nine times: the daemon handoff checks, the remote-client and remote-device checks, the request's actor, the kept +//! terminal of the key and the reply's incognito filter. Every connect, disconnect, attach, detach and capability +//! change of every client holds that lock, and an agent storm connects and disconnects about 120 times a second. +//! Under the L6 storm the fixed build still showed a daemon key part of p99 20-24 ms and max 74-223 ms (STATUS 7761). //! //! The test drives a key line through the real connection path (the line handler, the connection's dispatcher, the -//! `send` handler) while another thread holds one shared lock for 300 ms. A key that waits for the lock reaches its -//! PTY after about 300 ms; a key that does not wait reaches it in well under a millisecond. Two keys go in a row: a -//! lock taken after the PTY write (sizing, activity) delays the second key, which waits on the connection's -//! dispatcher behind the first. +//! `send` handler) while another thread holds one lock for 300 ms. A key that waits for the lock reaches its PTY after +//! about 300 ms; a key that does not wait reaches it in well under a millisecond. Two keys go in a row: a lock taken +//! after the PTY write delays the second key, which waits on the connection's dispatcher behind the first. -// The test terminal's in-process PTY runtime and the host proxy lock exist on Unix only (as for input_lane.rs). +// The test terminal's in-process PTY runtime is Unix-only (as for input_lane.rs). #![cfg(unix)] use super::*; @@ -22,10 +20,10 @@ use base64::Engine; use std::sync::mpsc; use std::time::Instant; -/// How long the other thread holds each lock. +/// How long the other thread holds the lock. const HOLD: Duration = Duration::from_millis(300); -/// A key that took longer than this waited for the held lock. Well above scheduling noise on a loaded fleet host, -/// well below `HOLD`. +/// A key that took longer than this waited for the held lock: well above scheduling noise on a loaded fleet host, +/// well below `HOLD` (the budget of input_lane.rs). const BUDGET: Duration = Duration::from_millis(150); /// Records when each write reached the test terminal's PTY input. @@ -101,7 +99,7 @@ impl TypingConnection { } } - /// Waits for the replies to `ids`, so the connection's dispatcher is idle again. + /// Waits for the replies to `ids` (each must be ok), so the connection's dispatcher is idle again. fn settle(&self, ids: &[&str]) { let deadline = Instant::now() + Duration::from_secs(5); let mut left: Vec<&str> = ids.to_vec(); @@ -109,10 +107,10 @@ impl TypingConnection { match self.outbound.try_pop() { Some(line) => { let value: Value = serde_json::from_str(&line).unwrap_or_default(); - if let Some(id) = value["id"].as_str() { - if value["type"] == "response" || value.get("ok").is_some() { - assert_eq!(value["ok"], true, "key {id} failed: {value}"); - } + if let Some(id) = value["id"].as_str() + && value.get("ok").is_some() + { + assert_eq!(value["ok"], true, "key {id} failed: {value}"); left.retain(|pending| *pending != id); } } @@ -129,45 +127,26 @@ impl TypingConnection { } } -/// Holds one shared lock on another thread until released or `HOLD` passed. +/// Holds one lock on another thread and runs `hold` while it holds it. type Holder = fn(&Arc, &dyn Fn()); -/// Every shared lock the key line may meet, each with a holder. `Mux.state` and the workspace registry are controls: -/// #31940 already took them off the key, so their rows pass on the base too. -fn hold_each() -> Vec<(&'static str, Holder)> { - let mut locks: Vec<(&'static str, Holder)> = Vec::new(); - locks.push(("client registry (ClientRegistry.state)", |mux, hold| { - let _held = mux.control_clients.state.lock().unwrap_or_else(|e| e.into_inner()); - hold(); - })); - locks.push(("link peers (RemoteRelayState.peers)", |mux, hold| { - let _held = mux.remote_relay().peers.lock().unwrap_or_else(|e| e.into_inner()); - hold(); - })); - locks.push(("host proxy authors (AgentSessions.proxy_authors)", |mux, hold| { - mux.control_clients.agent_sessions.hold_proxy_authors_for_test(hold); - })); - locks.push(("surface admission (ServerSurfaceOperationAdmission.state)", |mux, hold| { - let _held = mux.surface_operation_admission.state.lock().unwrap_or_else(|e| e.into_inner()); - hold(); - })); - locks.push(("creating input (CreatingInputs.terminals)", |mux, hold| { - mux.hold_creating_input_lock_for_test(hold); - })); - locks.push(("client sizing (Mux.client_sizing)", |mux, hold| { - mux.hold_client_sizing_lock_for_test(hold); - })); - locks.push(("activity (ActivityStream)", |mux, hold| mux.activity.hold_lock_for_test(hold))); - locks.push(("session (Mux.state)", |mux, hold| mux.with_state(|_| hold()))); - locks.push(("workspace registry (Mux.workspace_registry)", |mux, hold| { - let _held = mux.workspace_registry.lock().unwrap_or_else(|e| e.into_inner()); - hold(); - })); - locks +fn hold_client_registry(mux: &Arc, hold: &dyn Fn()) { + let _held = mux.control_clients.state.lock().unwrap_or_else(|e| e.into_inner()); + hold(); } -#[test] -fn a_key_waits_for_no_shared_lock() { +fn hold_session(mux: &Arc, hold: &dyn Fn()) { + mux.with_state(|_| hold()); +} + +fn hold_workspace_registry(mux: &Arc, hold: &dyn Fn()) { + let _held = mux.workspace_registry.lock().unwrap_or_else(|e| e.into_inner()); + hold(); +} + +/// Sends two keys while another thread holds each lock in turn, and fails when a key waited for one of them. +fn assert_keys_wait_for_none(locks: &[(&str, Holder)]) { + crate::rds_switch::set_input_lane_for_test(true); let connection = TypingConnection::open(); // The first key of a view may take the long way; every key after it must not wait. connection.submit("warm", b'w'); @@ -175,7 +154,7 @@ fn a_key_waits_for_no_shared_lock() { connection.settle(&["warm"]); let mut waits = Vec::new(); - for (index, (name, holder)) in hold_each().into_iter().enumerate() { + for (index, (name, holder)) in locks.iter().copied().enumerate() { let first = b'a' + u8::try_from(index * 2).unwrap_or(0); let second = first + 1; let (held, holding) = mpsc::channel(); @@ -213,9 +192,27 @@ fn a_key_waits_for_no_shared_lock() { .collect(); assert!( blocked.is_empty(), - "the person's keys waited for {} shared lock(s) held by another thread (budget {BUDGET:?}, held \ - {HOLD:?}): {}", + "the person's keys waited for {} lock(s) another thread held (budget {BUDGET:?}, held {HOLD:?}): {}", blocked.len(), blocked.join("; ") ); } + +/// Stage S1 step 1: the key line reads the facts of the session's connections from the client registry's published +/// projection, never through its lock. +#[test] +fn a_key_waits_for_no_client_registry_lock() { + assert_keys_wait_for_none(&[( + "client registry (ClientRegistry.state)", + hold_client_registry as Holder, + )]); +} + +/// #31940 and TYPING FIX 2 took these two off the key; this keeps them off on the stage's new path. +#[test] +fn a_key_waits_for_neither_the_session_nor_the_workspace_registry_lock() { + assert_keys_wait_for_none(&[ + ("session (Mux.state)", hold_session as Holder), + ("workspace registry (Mux.workspace_registry)", hold_workspace_registry as Holder), + ]); +} From f1ea1b6970929c6886336a28ce63a2c5f67c095f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:37:28 +0000 Subject: [PATCH 03/42] fix(cmux-tui-core): the person's key reads the client registry's published facts, never its lock (R-DAEMON-STATE S1 step 1) A key line took the client registry lock (ClientRegistry.state) about nine times: the daemon handoff checks, the remote-client and remote-device checks, the request's actor, the kept terminal of the key and the reply's incognito filter. Every connect, disconnect, attach, detach and capability change of every client holds that lock, so an agent storm held back the person's keys (red at 2472251831862: 300.4 ms behind a 300 ms hold; STATUS 9131). The registry is now a Published value (published.rs): writers still serialize on its mutex, and a guard that changed the value publishes an immutable ClientFacts projection (handoff state; per connection its transport, remote device flag, base actor and kept input terminals) with arc-swap when it drops, under the mutex, so projections follow the order of the changes. With the stage switch CMUX_TUI_RDS_INPUT=1, the line and key paths read the projection with one atomic load: daemon_handoff_in_progress/committed, transport_of (so is_remote_client), is_remote_device (refuse_line, line_to_recheck, prune_reply), connection_actor and attached_input_surface. A poisoned registry still fails closed (the projection comes in the error, as Mutex::lock gives its guard). With the switch off nothing changes. A projection costs O(connections) under the registry lock per change, bounded by the 64-connection cap; stage S2 makes it O(changed). arc-swap 1.9 is already in the lockfile and the shipped notices (through iroh-dns); cmux-tui-core now names it. Tests: rds_input_lane (a key waits for no client registry lock; the session and workspace registry locks stay off; a respawn under the same id sends the next key to the new shell), published.rs unit tests (a change publishes on drop, a read publishes nothing, a reader never waits for a writer, a poisoned value fails closed). The two remote-relay poison tests poison through the new type. --- cmux-tui/Cargo.lock | 1 + cmux-tui/Cargo.toml | 2 + cmux-tui/crates/cmux-tui-core/Cargo.toml | 1 + cmux-tui/crates/cmux-tui-core/src/lib.rs | 2 + .../crates/cmux-tui-core/src/published.rs | 193 ++++++++++++++++++ .../src/server/client_registry.rs | 78 ++++++- .../src/server/client_registry_views.rs | 12 ++ .../src/server/incognito_remote.rs | 5 + .../cmux-tui-core/src/server/origin_gate.rs | 10 + .../src/server/remote_relay/mod.rs | 11 +- .../src/server/remote_relay/tests.rs | 4 +- .../src/server/tests/rds_input_lane.rs | 36 ++++ 12 files changed, 346 insertions(+), 9 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/published.rs diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index aa899b1cf103..5dfc0d3a7c57 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -1803,6 +1803,7 @@ name = "cmux-tui-core" version = "1.0.0" dependencies = [ "anyhow", + "arc-swap", "base64", "cmux-agent-pane-policy", "cmux-api-compat", diff --git a/cmux-tui/Cargo.toml b/cmux-tui/Cargo.toml index 8883538ad3cc..73edc01a259d 100644 --- a/cmux-tui/Cargo.toml +++ b/cmux-tui/Cargo.toml @@ -329,6 +329,8 @@ notify = "8" iroh = { version = "1.0.3", default-features = false, features = ["fast-apple-datapath", "portmapper", "tls-ring", "platform-verifier"] } subtle = "2.6.1" parking_lot = "0.12" +# One-load published snapshots (cmux-tui-core `published.rs`, R-DAEMON-STATE); already in the tree through iroh-dns. +arc-swap = "1.9" wait-timeout = "0.2" # rusqlite 0.40's libsqlite3-sys build script requires Rust 1.95 without # declaring that MSRV. Keep the workspace's advertised Rust 1.91 support. diff --git a/cmux-tui/crates/cmux-tui-core/Cargo.toml b/cmux-tui/crates/cmux-tui-core/Cargo.toml index 31e2b8d7e81f..270132cb02cd 100644 --- a/cmux-tui/crates/cmux-tui-core/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-core/Cargo.toml @@ -54,6 +54,7 @@ cmux-link.workspace = true getrandom.workspace = true rusqlite.workspace = true parking_lot = { workspace = true, features = ["arc_lock"] } +arc-swap.workspace = true fs4.workspace = true sha2.workspace = true socket2.workspace = true diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index 3ea4dd108646..a4a29c417603 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -70,6 +70,8 @@ pub mod provider_management; mod pty_input_room; #[cfg(unix)] mod pty_write; +/// A mutex-guarded value whose readers load a published projection without the lock (R-DAEMON-STATE). +mod published; mod terminal_title; use cmux_tui_remote_access::remote_relay_state; mod login_shell; diff --git a/cmux-tui/crates/cmux-tui-core/src/published.rs b/cmux-tui/crates/cmux-tui-core/src/published.rs new file mode 100644 index 000000000000..08f707450ede --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/published.rs @@ -0,0 +1,193 @@ +//! A value that one thread at a time changes and any thread reads without waiting +//! (handoff/rewrites/R-DAEMON-STATE/DESIGN.md, rule R2: readers never wait). +//! +//! Writers serialize on the value's mutex, as before. A guard that changed the value (it was +//! borrowed mutably) publishes an immutable projection of the value when it drops, while it still +//! holds the mutex, so projections are published in the order of the changes. A reader loads the +//! newest projection with one atomic load (`arc-swap`): it never takes the mutex, so no reader +//! waits for a writer and no writer waits for a reader. +//! +//! The projection is the only copy a reader sees, and only the guard writes it: no code path can +//! change the value without publishing the change, because every change goes through the guard. + +use std::ops::{Deref, DerefMut}; +use std::sync::{Arc, LockResult, Mutex, MutexGuard, PoisonError}; +#[cfg(test)] +use std::sync::{TryLockError, TryLockResult}; + +use arc_swap::ArcSwap; + +/// A mutex-guarded value with a lock-free published projection (see the module docs). +pub(crate) struct Published { + value: Mutex, + projection: ArcSwap, + project: fn(&T) -> S, +} + +impl Published { + /// `project` makes the immutable projection that readers load. It runs under the mutex after + /// each change, so it must read only and stay small. + pub(crate) fn new(value: T, project: fn(&T) -> S) -> Self { + let projection = ArcSwap::from_pointee(project(&value)); + Self { value: Mutex::new(value), projection, project } + } + + /// The projection of the newest finished change: one atomic load, never a wait. When a writer + /// panicked while it held the value (the mutex is poisoned), the projection comes in the error, + /// as `std::sync::Mutex::lock` gives its guard, so a caller can fail closed. + pub(crate) fn load(&self) -> LockResult>> { + let projection = self.projection.load(); + if self.value.is_poisoned() { Err(PoisonError::new(projection)) } else { Ok(projection) } + } + + /// The value itself, for a change or for a read that needs more than the projection. A + /// poisoned mutex gives its guard in the error, as `std::sync::Mutex::lock` does. + pub(crate) fn lock(&self) -> LockResult> { + match self.value.lock() { + Ok(guard) => Ok(PublishGuard { owner: self, guard, changed: false }), + Err(poison) => Err(PoisonError::new(PublishGuard { + owner: self, + guard: poison.into_inner(), + changed: false, + })), + } + } +} + +#[cfg(test)] +impl Published { + /// The value if no thread holds it now (tests that prove a hold). + pub(crate) fn try_lock(&self) -> TryLockResult> { + match self.value.try_lock() { + Ok(guard) => Ok(PublishGuard { owner: self, guard, changed: false }), + Err(TryLockError::WouldBlock) => Err(TryLockError::WouldBlock), + Err(TryLockError::Poisoned(poison)) => { + Err(TryLockError::Poisoned(PoisonError::new(PublishGuard { + owner: self, + guard: poison.into_inner(), + changed: false, + }))) + } + } + } +} + +/// A held [`Published`] value. A mutable borrow marks it changed; it then publishes the new +/// projection when it drops, before the mutex is released (fields drop after `Drop::drop`). +pub(crate) struct PublishGuard<'a, T, S> { + owner: &'a Published, + guard: MutexGuard<'a, T>, + changed: bool, +} + +impl Deref for PublishGuard<'_, T, S> { + type Target = T; + + fn deref(&self) -> &T { + &self.guard + } +} + +impl DerefMut for PublishGuard<'_, T, S> { + fn deref_mut(&mut self) -> &mut T { + self.changed = true; + &mut self.guard + } +} + +impl Drop for PublishGuard<'_, T, S> { + fn drop(&mut self) { + if self.changed { + self.owner.projection.store(Arc::new((self.owner.project)(&*self.guard))); + } + } +} + +#[cfg(test)] +mod tests { + use super::Published; + + #[derive(Default)] + struct Counter { + value: u64, + reads: u64, + } + + fn project(counter: &Counter) -> u64 { + counter.value + } + + #[test] + fn a_change_is_published_when_its_guard_drops_and_a_read_publishes_nothing() { + let published = Published::new(Counter::default(), project); + assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 0); + { + let mut guard = published.lock().unwrap_or_else(|poison| poison.into_inner()); + guard.value = 7; + // Not yet: the guard still holds the change. + assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 0); + } + assert_eq!( + **published.load().unwrap_or_else(|poison| poison.into_inner()), + 7, + "the change was published when its guard dropped" + ); + let before = published.load_full_for_test(); + { + let guard = published.lock().unwrap_or_else(|poison| poison.into_inner()); + assert_eq!(guard.reads, 0); + } + assert!( + std::sync::Arc::ptr_eq(&before, &published.load_full_for_test()), + "a guard that only read published nothing" + ); + } + + #[test] + fn a_reader_does_not_wait_for_a_writer_that_holds_the_value() { + let published = std::sync::Arc::new(Published::new(Counter::default(), project)); + let (held, holding) = std::sync::mpsc::channel(); + let (release, released) = std::sync::mpsc::channel::<()>(); + let writer = std::thread::spawn({ + let published = published.clone(); + move || { + let mut guard = published.lock().unwrap_or_else(|poison| poison.into_inner()); + guard.value = 1; + let _ = held.send(()); + let _ = released.recv_timeout(std::time::Duration::from_secs(5)); + } + }); + let _ = holding.recv(); + let started = std::time::Instant::now(); + let seen = **published.load().unwrap_or_else(|poison| poison.into_inner()); + let waited = started.elapsed(); + let _ = release.send(()); + let _ = writer.join(); + assert_eq!(seen, 0, "the reader saw the last finished change, not the one in progress"); + assert!(waited < std::time::Duration::from_millis(100), "the reader waited {waited:?}"); + assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 1); + } + + #[test] + fn a_poisoned_value_gives_its_projection_in_the_error() { + let published = Published::new(Counter::default(), project); + assert!(published.load().is_ok()); + published.poison_for_test(); + assert!(published.load().is_err(), "a reader can fail closed on a poisoned value"); + } + + impl Published { + fn load_full_for_test(&self) -> std::sync::Arc { + self.projection.load_full() + } + + /// Poisons the value: a panic while it is held (fail-closed tests). + pub(crate) fn poison_for_test(&self) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _held = self.value.lock(); + panic!("poisons the value for a fail-closed test"); + })); + assert!(self.value.is_poisoned()); + } + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs b/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs index c6820089d11b..84a73dff3ae6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs @@ -326,6 +326,69 @@ pub(super) struct ClientRegistryState { pub(super) daemon_handoff: Option, } +/// What a connection's own line and key paths read about the session's connections, published after every change +/// of the client registry and read with one atomic load, never through its lock (R-DAEMON-STATE stage S1, +/// handoff/rewrites/R-DAEMON-STATE/DESIGN.md section 2.6). Every connect, disconnect, attach and detach of every +/// client holds the registry lock; a person's key must not wait for any of them. +pub(super) struct ClientFacts { + /// The daemon handoff, when one is reserved: `true` once it was acknowledged (committed). + pub(super) handoff: Option, + pub(super) clients: HashMap, +} + +/// One connection's facts in [`ClientFacts`]. +pub(super) struct ConnectionFacts { + pub(super) transport: ClientTransport, + /// `ClientRecord::remote_device`. + pub(super) remote_device: bool, + /// The connection's base actor (`ConnectionOrigin::actor`). + pub(super) actor: crate::workspace_registry::Actor, + /// The terminal the connection's keys to each attached surface go to (`AttachedSurface::input`). + pub(super) inputs: Vec, +} + +/// `AttachedSurface::input` of one attached surface, with whether the connection can type into it. +pub(super) struct KeptInput { + pub(super) surface: SurfaceId, + pub(super) terminal: Weak, + /// The route generation of `surface` when the terminal was kept (`route_table`). + pub(super) generation: u64, + pub(super) can_type: bool, +} + +impl ClientFacts { + /// The projection of `state` (runs under the registry lock after each change). + fn of(state: &ClientRegistryState) -> Self { + let handoff = state + .daemon_handoff + .map(|reservation| matches!(reservation, DaemonHandoffReservation::Committed(_))); + let clients = state + .clients + .iter() + .map(|(client, record)| { + let inputs = record + .attached + .iter() + .map(|(surface, attached)| KeptInput { + surface: *surface, + terminal: attached.input.0.clone(), + generation: attached.input.1, + can_type: attached.can_type(), + }) + .collect(); + let facts = ConnectionFacts { + transport: record.transport, + remote_device: record.remote_device, + actor: record.origin.actor(), + inputs, + }; + (*client, facts) + }) + .collect(); + Self { handoff, clients } + } +} + pub(crate) struct ClientRegistry { /// Called when a surface loses its last attached client (the idle-close /// reaper starts that terminal's unattached period). @@ -357,7 +420,9 @@ pub(crate) struct ClientRegistry { pub(super) next_id: AtomicU64, pub(super) resource_stream_admission: Arc, pub(super) resource_wait_admission: Arc, - pub(super) state: Mutex, + /// Every connection's record. A change publishes the [`ClientFacts`] projection, which a connection's + /// own line and key paths read without this lock (R-DAEMON-STATE stage S1). + pub(super) state: crate::published::Published, /// Who created each live incognito workspace (incognito_orphans.rs). pub(super) incognito_creators: super::incognito_orphans::IncognitoCreators, } @@ -426,7 +491,10 @@ impl ClientRegistry { RESOURCE_WAITS_PER_CLIENT_CAPACITY, RESOURCE_WAITS_SERVER_CAPACITY, ), - state: Mutex::new(ClientRegistryState::default()), + state: crate::published::Published::new( + ClientRegistryState::default(), + ClientFacts::of, + ), incognito_creators: Default::default(), } } @@ -500,10 +568,16 @@ impl ClientRegistry { } pub(crate) fn daemon_handoff_in_progress(&self) -> bool { + if crate::rds_switch::input_lane() { + return self.state.load().recover_poison().handoff.is_some(); + } self.state.lock().recover_poison().daemon_handoff.is_some() } pub(crate) fn daemon_handoff_committed(&self) -> bool { + if crate::rds_switch::input_lane() { + return self.state.load().recover_poison().handoff == Some(true); + } matches!( self.state.lock().recover_poison().daemon_handoff, Some(DaemonHandoffReservation::Committed(_)) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs b/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs index 5775c76c244a..fe7aab886d9b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs @@ -395,6 +395,18 @@ impl ClientRegistry { client: u64, surface: SurfaceId, ) -> Option<(Arc, bool)> { + if crate::rds_switch::input_lane() { + // The registry's published facts: no lock that a connect, attach or detach of any client holds. + let facts = self.state.load().recover_poison(); + let kept = + facts.clients.get(&client)?.inputs.iter().find(|kept| kept.surface == surface)?; + if kept.generation != crate::route_table::route_generation(surface) { + return None; + } + let (input, can_type) = (kept.terminal.upgrade()?, kept.can_type); + drop(facts); + return (!input.is_dead()).then_some((input, can_type)); + } let state = self.state.lock().recover_poison(); let attached = state.clients.get(&client)?.attached.get(&surface)?; let (kept, generation) = &attached.input; diff --git a/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs b/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs index 629f6fb37f51..af729fcb04fe 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs @@ -181,6 +181,11 @@ pub(super) struct Hidden { /// Whether `client` is a remote device's connection. pub(super) fn is_remote_device(mux: &Mux, client: u64) -> bool { + if crate::rds_switch::input_lane() { + // The published facts, never the registry lock (R-DAEMON-STATE S1). + let facts = mux.control_clients.state.load().recover_poison(); + return facts.clients.get(&client).is_some_and(|connection| connection.remote_device); + } let state = mux.control_clients.state.lock().recover_poison(); state.clients.get(&client).is_some_and(|record| record.remote_device) } diff --git a/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs b/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs index 8d672667dd14..81a63ab16bc4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs @@ -220,6 +220,16 @@ fn check_pairing_accept( /// The actor of a durable mutation that `client` asks for on the legacy /// control protocol. A client with no record is never the local user. pub(super) fn connection_actor(mux: &Mux, client: u64) -> Actor { + if crate::rds_switch::input_lane() { + // The published facts, never the registry lock (R-DAEMON-STATE S1). + let facts = mux.control_clients.state.load().recover_poison(); + let Some(connection) = facts.clients.get(&client) else { + return Actor::Peer { id: "unregistered".to_string() }; + }; + let (transport, local) = (connection.transport, connection.actor.clone()); + drop(facts); + return peer_or_local(mux, client, transport, local); + } let state = mux.control_clients.state.lock().recover_poison(); let Some(record) = state.clients.get(&client) else { return Actor::Peer { id: "unregistered".to_string() }; diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs index 1b382ade124e..75546f05460e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs @@ -106,11 +106,12 @@ impl super::ClientRegistry { /// A poisoned registry lock also gives `None`, which every caller treats /// as remote (fail closed). pub(super) fn transport_of(&self, client: u64) -> Option { - lock_checked(&self.state, RelayLock::Clients) - .ok()? - .clients - .get(&client) - .map(|record| record.transport) + if crate::rds_switch::input_lane() { + // The published facts, never the registry lock (R-DAEMON-STATE S1); a poisoned registry still + // fails closed. + return self.state.load().ok()?.clients.get(&client).map(|facts| facts.transport); + } + self.state.lock().ok()?.clients.get(&client).map(|record| record.transport) } /// True for a connection that came through the remote entry (tests; diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs index 8f61961bd9c1..0586005c8e68 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs @@ -1023,7 +1023,7 @@ fn a_poisoned_peers_lock_gives_no_principal_and_no_local_trust() { #[test] fn a_poisoned_registry_lock_trusts_no_client_as_local() { let fixture = fixture(); - poison(&fixture.mux.control_clients.state); + fixture.mux.control_clients.state.poison_for_test(); assert!(fixture.mux.control_clients.transport_of(fixture.local).is_none()); assert!(fixture.mux.is_remote_client(fixture.local)); } @@ -1134,7 +1134,7 @@ fn a_remote_bind_on_a_poisoned_bindings_lock_refuses_without_a_panic() { fn a_poisoned_registry_trusts_no_unix_client_and_still_removes_a_record() { let fixture = fixture(); let one = remote(&fixture, "inst_1", OWNER); - poison(&fixture.mux.control_clients.state); + fixture.mux.control_clients.state.poison_for_test(); assert!(!fixture.mux.control_clients.is_unix(fixture.local)); assert_eq!(fixture.mux.principal(fixture.local), None); assert!(fixture.mux.control_clients.remove(one).is_some()); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index 095cdc3c8f66..7b3e26e52f70 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -216,3 +216,39 @@ fn a_key_waits_for_neither_the_session_nor_the_workspace_registry_lock() { ("workspace registry (Mux.workspace_registry)", hold_workspace_registry as Holder), ]); } + +/// The published facts follow the routing: after a respawn under the same placement id, the next key reaches the new +/// shell, never the replaced one (the route rule of #31940, on the stage's path). +#[test] +fn after_a_respawn_the_next_key_reaches_the_new_terminal_on_the_input_lane() { + crate::rds_switch::set_input_lane_for_test(true); + let connection = TypingConnection::open(); + connection.submit("before", b'a'); + assert!(connection.arrival(b'a').is_some(), "the first key never reached its terminal"); + connection.settle(&["before"]); + + let removed = connection + .mux + .remove_surface_runtime_for_test(connection.surface) + .expect("the placement had a runtime"); + let respawned = + crate::Surface::spawn_for_test(connection.surface, SurfaceOptions::default(), Weak::new()) + .unwrap(); + let (sent, input) = mpsc::channel(); + respawned.replace_input_writer_for_test(Box::new(TimedInput(sent))); + connection.mux.insert_surface_runtime_for_test(respawned); + + connection.submit("after", b'b'); + let reached = input + .recv_timeout(Duration::from_secs(5)) + .ok() + .is_some_and(|(_, bytes)| bytes.contains(&b'b')); + connection.settle(&["after"]); + assert!(reached, "the key after the respawn did not reach the new shell"); + assert!( + connection.input.try_recv().is_err(), + "the key after the respawn went to the replaced terminal" + ); + drop(removed); + connection.close(); +} From 590cb119161cc16ae3e78b4e7f6fab9bd21b77db Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:29:16 +0000 Subject: [PATCH 04/42] fix(cmux-tui-core): the key reads the session's published routes; one stage switch per mux (R-DAEMON-STATE S1 step 1) The hosted run of f1ea1b69709 (https://github.com/manaflow-ai/cmux/actions/runs/41655768031) still failed both lane tests: the key waited 300 ms for the client registry and, new, for Mux.state. Two causes: - The key still used the kept terminal of #31940. That cache is validated by process-wide route generations, and cargo test runs the tests of one binary as threads of one process: another test's mux moved the same generation slot (ids start at 1 in every mux), the kept terminal looked stale, and the key fell back to the locked lookup and then to remember_attached_input (registry lock). The stage switch was a process-wide static too. - So the lane must not depend on a cache with process-wide invalidation at all. Now, with CMUX_TUI_RDS_INPUT=1: - The session's three routing tables (surfaces, terminal_catalog, terminal_catalog_by_runtime) are copy-on-write behind an Arc. A state guard that changed one publishes a RouteSnapshot (three Arc clones) with arc-swap when it drops, under the lock, so publications follow the changes; the snapshot it replaced is freed after the lock is released. Mux::surface reads the snapshot with one atomic load. A respawn, close, move, rollback or projection is seen once its guard is released, as a reader that waited for the lock would see it. Nothing is kept per client, so nothing is invalidated. - input_surface on the lane: the published routes give the terminal, the registry's published facts say whether the client types into it (types_into: the person's input QoS mark). attached_input_surface and remember_attached_input stay only on the old path. - The switch is per mux (rds_switch::InputLaneSwitch, read once from the environment when the mux is built; Mux::set_input_lane_for_test in tests), and it also decides whether anything is published: off, the registry builds no projection and the state holds no route snapshot, so a route write never copies a shared table and the off arm costs what the base costs. Cost on the lane: a guard's first route write copies that table once (O(routes)), bounded by one copy per changing guard; State::clone no longer deep-copies the three tables. Stage S2 makes both O(log n). --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 12 +++ .../crates/cmux-tui-core/src/mux/construct.rs | 7 +- .../cmux-tui-core/src/mux/signaled_mutex.rs | 68 ++++++++++++++-- .../crates/cmux-tui-core/src/published.rs | 37 +++++++-- .../crates/cmux-tui-core/src/rds_switch.rs | 49 +++++------- .../crates/cmux-tui-core/src/route_table.rs | 80 +++++++++++++++++-- .../src/server/client_registry.rs | 54 ++++++++----- .../src/server/client_registry_views.rs | 12 --- .../src/server/cmd_terminal_io.rs | 10 +++ .../src/server/incognito_remote.rs | 2 +- .../cmux-tui-core/src/server/origin_gate.rs | 2 +- .../src/server/remote_relay/mod.rs | 2 +- .../src/server/tests/rds_input_lane.rs | 3 +- 13 files changed, 248 insertions(+), 90 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 9d352b3dcbb1..841d403d5a0a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -929,6 +929,11 @@ impl Mux { #[track_caller] pub fn surface(&self, id: SurfaceId) -> Option> { + if self.control_clients.input_lane.on() { + // The routing tables of the last finished change, with no lock (R-DAEMON-STATE stage S1): the same + // lookup, never a wait behind a list, a snapshot or a durable mutation that holds the session lock. + return self.state.routes().surface(id); + } let state = self.state.lock().unwrap(); state.surfaces.get(&id).or_else(|| state.terminal_runtime_by_id(id)).cloned() } @@ -946,6 +951,13 @@ impl Mux { self.state.lock().unwrap().surfaces.remove(&id) } + /// Turns R-DAEMON-STATE stage S1 (the input lane) on or off for this mux (tests). + #[cfg(test)] + pub(crate) fn set_input_lane_for_test(&self, on: bool) { + self.state.set_route_publishing_for_test(on); + self.control_clients.set_input_lane_for_test(on); + } + #[cfg(test)] pub(crate) fn insert_surface_runtime_for_test(&self, surface: Arc) { let previous = self.state.lock().unwrap().surfaces.insert(surface.id, surface); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs index 382358d76abf..b0d7dcfb362b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs @@ -202,6 +202,9 @@ impl Mux { let launch_identity = crate::launch_credential::LaunchIdentity::load( registry.session_journal_database_path().as_deref().and_then(Path::parent), ); + // One stage S1 switch for the mux (R-DAEMON-STATE): the state's routes and the client registry's facts. + let input_lane = crate::rds_switch::InputLaneSwitch::from_env(); + let publish_routes = input_lane.on(); let mux = Arc::new(Mux { launch_identity, registry_connection: registry.connection.clone(), @@ -213,7 +216,7 @@ impl Mux { registry_identity, resource_projection_stats, started_at: Instant::now(), - state: signaled_mutex::StateMutex::new(state), + state: signaled_mutex::StateMutex::new(state, publish_routes), subscribers: MuxEventBroadcaster::default(), config_reload: Mutex::new(ConfigReloadState::default()), config_reload_changed: Condvar::new(), @@ -409,7 +412,7 @@ impl Mux { exit_settles: Arc::default(), terminal_respawns: terminal_respawn::TerminalRespawns::from_env(), daemon_shutdown_waker: Mutex::new(None), - control_clients: crate::server::ClientRegistry::new(), + control_clients: crate::server::ClientRegistry::new(input_lane), #[cfg(unix)] agent_transfers: Default::default(), #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs b/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs index 26dd877f32ed..fe69bf248b24 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs @@ -213,6 +213,12 @@ impl Drop for SignaledMutexGuard<'_, T> { pub(crate) struct StateMutex { value: Mutex, stats: crate::diagnostics::LockStats, + /// The routing tables of the last finished change (R-DAEMON-STATE stage S1): a guard that changed them publishes + /// them when it drops, under the lock, so `Mux::surface` reads them with one atomic load and never waits. + routes: arc_swap::ArcSwap, + /// Whether guards publish `routes` (the stage S1 switch). Off, `routes` holds no table of the state, so a route + /// write never copies a table a snapshot shares. Read and written only under the lock. + publish_routes: std::sync::atomic::AtomicBool, } /// A held mux state lock. @@ -221,21 +227,27 @@ pub(crate) struct StateGuard<'a> { /// the body), so the release is recorded while the lock is held. guard: MutexGuard<'a, State>, stats: &'a crate::diagnostics::LockStats, + /// The mutex, for `Drop::drop` to publish the routing tables when they changed (R-DAEMON-STATE stage S1). + owner: &'a StateMutex, site: crate::diagnostics::LockSite, waited: std::time::Duration, acquired_at: Instant, /// Lock rank `MuxState` (crate::lock_rank), released after the lock. _rank: HeldRank, + /// The routing snapshot this guard replaced. It drops after `guard` (fields drop in order), so the tables and + /// surfaces it may hold last are freed after the lock is released, never under it. + retired: Option>, } impl<'a> StateGuard<'a> { fn new( guard: MutexGuard<'a, State>, - stats: &'a crate::diagnostics::LockStats, + owner: &'a StateMutex, site: crate::diagnostics::LockSite, waited_from: Instant, blocker: Option, ) -> StateGuard<'a> { + let stats = &owner.stats; // The journal writer must never take state (counted in release): // the lock order puts it after the writer's rank. let on_writer = crate::workspace_registry::registry_connection::in_journal_writer_commit(); @@ -251,10 +263,12 @@ impl<'a> StateGuard<'a> { StateGuard { guard, stats, + owner, site, waited, acquired_at: Instant::now(), _rank: HeldRank::record(LockRank::MuxState, STATE_LOCK_NAME), + retired: None, } } } @@ -275,6 +289,15 @@ pub(crate) fn state_locks_taken_on_this_thread() -> u64 { impl Drop for StateGuard<'_> { fn drop(&mut self) { + // Publish the routing tables a change replaced, while the lock is still held, so publications follow the + // order of the changes (R-DAEMON-STATE stage S1). A guard that changed no route compares three pointers. + let owner = self.owner; + if owner.publish_routes.load(std::sync::atomic::Ordering::Relaxed) + && !owner.routes.load().is_current(&self.guard) + { + let published = std::sync::Arc::new(crate::route_table::RouteSnapshot::of(&self.guard)); + self.retired = Some(owner.routes.swap(published)); + } // Record the release while the lock is still held, so the next // holder's record is never erased by this older unlock. self.stats.released(self.site, self.acquired_at.elapsed(), self.waited); @@ -297,8 +320,39 @@ impl DerefMut for StateGuard<'_> { } impl StateMutex { - pub(crate) fn new(state: State) -> Self { - Self { value: Mutex::new(state), stats: crate::diagnostics::LockStats::new() } + /// `publish_routes` is the mux's stage S1 switch (R-DAEMON-STATE). + pub(crate) fn new(state: State, publish_routes: bool) -> Self { + let routes = arc_swap::ArcSwap::from_pointee(if publish_routes { + crate::route_table::RouteSnapshot::of(&state) + } else { + crate::route_table::RouteSnapshot::default() + }); + Self { + value: Mutex::new(state), + stats: crate::diagnostics::LockStats::new(), + routes, + publish_routes: std::sync::atomic::AtomicBool::new(publish_routes), + } + } + + /// Turns route publication on or off (tests); on, the current tables are published first, under the lock. + #[cfg(test)] + pub(crate) fn set_route_publishing_for_test(&self, on: bool) { + let state = self.value.lock().unwrap_or_else(PoisonError::into_inner); + let snapshot = if on { + crate::route_table::RouteSnapshot::of(&state) + } else { + crate::route_table::RouteSnapshot::default() + }; + self.routes.store(std::sync::Arc::new(snapshot)); + self.publish_routes.store(on, std::sync::atomic::Ordering::Relaxed); + } + + /// The routing tables as of the last finished change: one atomic load, never a wait (R-DAEMON-STATE stage S1). + pub(crate) fn routes( + &self, + ) -> arc_swap::Guard> { + self.routes.load() } /// Hold and wait telemetry by site (`server-stats` `locks.state`). @@ -313,10 +367,10 @@ impl StateMutex { let waited_from = Instant::now(); let blocker = self.stats.wait_started(); match self.value.lock() { - Ok(guard) => Ok(StateGuard::new(guard, &self.stats, site, waited_from, blocker)), + Ok(guard) => Ok(StateGuard::new(guard, self, site, waited_from, blocker)), Err(poison) => Err(PoisonError::new(StateGuard::new( poison.into_inner(), - &self.stats, + self, site, waited_from, blocker, @@ -331,10 +385,10 @@ impl StateMutex { let site = std::panic::Location::caller(); let waited_from = Instant::now(); match self.value.try_lock() { - Ok(guard) => Ok(StateGuard::new(guard, &self.stats, site, waited_from, None)), + Ok(guard) => Ok(StateGuard::new(guard, self, site, waited_from, None)), Err(TryLockError::WouldBlock) => Err(TryLockError::WouldBlock), Err(TryLockError::Poisoned(poison)) => Err(TryLockError::Poisoned(PoisonError::new( - StateGuard::new(poison.into_inner(), &self.stats, site, waited_from, None), + StateGuard::new(poison.into_inner(), self, site, waited_from, None), ))), } } diff --git a/cmux-tui/crates/cmux-tui-core/src/published.rs b/cmux-tui/crates/cmux-tui-core/src/published.rs index 08f707450ede..0bbc9e386a4a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/published.rs +++ b/cmux-tui/crates/cmux-tui-core/src/published.rs @@ -9,8 +9,13 @@ //! //! The projection is the only copy a reader sees, and only the guard writes it: no code path can //! change the value without publishing the change, because every change goes through the guard. +//! +//! Publishing is on or off for the value's life (a stage switch decides it when the value is made; +//! tests can turn it on). Off, a guard builds no projection, so a value nobody reads lock-free costs +//! what a plain mutex costs. use std::ops::{Deref, DerefMut}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, LockResult, Mutex, MutexGuard, PoisonError}; #[cfg(test)] use std::sync::{TryLockError, TryLockResult}; @@ -22,14 +27,31 @@ pub(crate) struct Published { value: Mutex, projection: ArcSwap, project: fn(&T) -> S, + /// Whether a changing guard publishes; read and written only under `value`'s lock. + publishing: AtomicBool, } impl Published { /// `project` makes the immutable projection that readers load. It runs under the mutex after - /// each change, so it must read only and stay small. - pub(crate) fn new(value: T, project: fn(&T) -> S) -> Self { + /// each change while `publishing` is on, so it must read only and stay small. + pub(crate) fn new(value: T, project: fn(&T) -> S, publishing: bool) -> Self { let projection = ArcSwap::from_pointee(project(&value)); - Self { value: Mutex::new(value), projection, project } + Self { + value: Mutex::new(value), + projection, + project, + publishing: AtomicBool::new(publishing), + } + } + + /// Turns publishing on or off; on, the current value is published first, under the lock (tests). + #[cfg(test)] + pub(crate) fn set_publishing_for_test(&self, on: bool) { + let value = self.value.lock().unwrap_or_else(PoisonError::into_inner); + if on { + self.projection.store(Arc::new((self.project)(&*value))); + } + self.publishing.store(on, Ordering::Relaxed); } /// The projection of the newest finished change: one atomic load, never a wait. When a writer @@ -97,7 +119,8 @@ impl DerefMut for PublishGuard<'_, T, S> { impl Drop for PublishGuard<'_, T, S> { fn drop(&mut self) { - if self.changed { + // The flag is read under the lock this guard holds, where it is also written. + if self.changed && self.owner.publishing.load(Ordering::Relaxed) { self.owner.projection.store(Arc::new((self.owner.project)(&*self.guard))); } } @@ -119,7 +142,7 @@ mod tests { #[test] fn a_change_is_published_when_its_guard_drops_and_a_read_publishes_nothing() { - let published = Published::new(Counter::default(), project); + let published = Published::new(Counter::default(), project, true); assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 0); { let mut guard = published.lock().unwrap_or_else(|poison| poison.into_inner()); @@ -145,7 +168,7 @@ mod tests { #[test] fn a_reader_does_not_wait_for_a_writer_that_holds_the_value() { - let published = std::sync::Arc::new(Published::new(Counter::default(), project)); + let published = std::sync::Arc::new(Published::new(Counter::default(), project, true)); let (held, holding) = std::sync::mpsc::channel(); let (release, released) = std::sync::mpsc::channel::<()>(); let writer = std::thread::spawn({ @@ -170,7 +193,7 @@ mod tests { #[test] fn a_poisoned_value_gives_its_projection_in_the_error() { - let published = Published::new(Counter::default(), project); + let published = Published::new(Counter::default(), project, true); assert!(published.load().is_ok()); published.poison_for_test(); assert!(published.load().is_err(), "a reader can fail closed on a poisoned value"); diff --git a/cmux-tui/crates/cmux-tui-core/src/rds_switch.rs b/cmux-tui/crates/cmux-tui-core/src/rds_switch.rs index 8c7a1a934c5e..9f1e42834936 100644 --- a/cmux-tui/crates/cmux-tui-core/src/rds_switch.rs +++ b/cmux-tui/crates/cmux-tui-core/src/rds_switch.rs @@ -1,35 +1,30 @@ //! The runtime switches of the R-DAEMON-STATE rewrite (handoff/rewrites/R-DAEMON-STATE/DESIGN.md, -//! section 3): one per stage, read once from the daemon's environment, deleted together with the -//! old path when the stage passes. A switch only chooses between the stage's new path and the old -//! one; both stay correct, so an A/B can run either arm in the same build. +//! section 3): one per stage, read once from the daemon's environment when the mux is built, deleted +//! together with the old path when the stage passes. A switch only chooses between the stage's new +//! path and the old one; both stay correct, so an A/B can run either arm in the same build. Each mux +//! owns its switches, so a test that turns one on changes no other test's mux. -use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; -const UNREAD: u8 = 0; -const OFF: u8 = 1; -const ON: u8 = 2; - -/// Stage S1, the input lane: a connection's lines and the person's keys read the facts of the -/// session's connections from a published snapshot, never through a shared lock. +/// Stage S1, the input lane: a connection's lines and the person's keys read the published facts of +/// the session's connections and its routing tables, never through a shared lock. /// `CMUX_TUI_RDS_INPUT=1` turns it on. -static INPUT_LANE: AtomicU8 = AtomicU8::new(UNREAD); +pub(crate) struct InputLaneSwitch(AtomicBool); -/// Whether stage S1 (the input lane) is on. The environment is read once; a race of two first -/// reads only reads it twice. -pub(crate) fn input_lane() -> bool { - match INPUT_LANE.load(Ordering::Relaxed) { - ON => true, - OFF => false, - _ => { - let on = std::env::var("CMUX_TUI_RDS_INPUT").is_ok_and(|value| value == "1"); - INPUT_LANE.store(if on { ON } else { OFF }, Ordering::Relaxed); - on - } +impl InputLaneSwitch { + /// The switch as the daemon's environment sets it. + pub(crate) fn from_env() -> Self { + Self(AtomicBool::new(std::env::var("CMUX_TUI_RDS_INPUT").is_ok_and(|value| value == "1"))) } -} -/// Turns stage S1 on or off for this test process (nextest runs each test in its own process). -#[cfg(test)] -pub(crate) fn set_input_lane_for_test(on: bool) { - INPUT_LANE.store(if on { ON } else { OFF }, Ordering::Relaxed); + /// Whether stage S1 is on for this mux. + pub(crate) fn on(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + + /// Turns stage S1 on or off for this mux (tests). + #[cfg(test)] + pub(crate) fn set_for_test(&self, on: bool) { + self.0.store(on, Ordering::Relaxed); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/route_table.rs b/cmux-tui/crates/cmux-tui-core/src/route_table.rs index c44fea6ded43..9e527eb372bb 100644 --- a/cmux-tui/crates/cmux-tui-core/src/route_table.rs +++ b/cmux-tui/crates/cmux-tui-core/src/route_table.rs @@ -28,6 +28,13 @@ //! //! The generations are process-wide: a daemon runs one `Mux`, and the test runner (nextest) runs each test in its own //! process, so no other mux moves them under a test. +//! +//! R-DAEMON-STATE stage S1 (`CMUX_TUI_RDS_INPUT`, handoff/rewrites/R-DAEMON-STATE/DESIGN.md section 2.6) replaces the +//! kept terminal and its generations with a published snapshot of the three tables ([`RouteSnapshot`]): each table +//! is copy-on-write behind an `Arc`, and the session state's guard publishes the tables it changed when it drops, +//! while it still holds the lock. A key reads the snapshot with one atomic load, so it finds the terminal its id +//! routes to now (a respawn, close, move, rollback or projection included) and never takes the session lock. There +//! is no cache to invalidate. The generations stay for the old path until the stage's switch is deleted. use std::collections::HashMap; use std::hash::Hash; @@ -85,17 +92,19 @@ impl RouteKey> for TerminalPublicId { } } -/// A routing map whose writes move the [`route_generation`] of the ids they route. +/// A routing map whose writes move the [`route_generation`] of the ids they route. The map is copy-on-write behind an +/// `Arc`, so the session state's guard can publish it with no copy ([`RouteSnapshot`]); the first write after a +/// publication copies the map once. #[derive(Clone, Debug)] -pub struct RouteTable(HashMap); +pub struct RouteTable(Arc>); impl Default for RouteTable { fn default() -> Self { - Self(HashMap::new()) + Self(Arc::new(HashMap::new())) } } -impl, V> RouteTable { +impl, V: Clone> RouteTable { pub fn new() -> Self { Self::default() } @@ -110,23 +119,78 @@ impl, V> RouteTable { if let Some(id) = unrouted.filter(|id| Some(*id) != routed) { route_changed(id); } - self.0.insert(key, value) + Arc::make_mut(&mut self.0).insert(key, value) } pub fn remove(&mut self, key: &K) -> Option { - let removed = self.0.remove(key)?; + if !self.0.contains_key(key) { + return None; + } + let removed = Arc::make_mut(&mut self.0).remove(key)?; if let Some(id) = key.routed_id(&removed) { route_changed(id); } Some(removed) } - /// Room for `additional` more entries (a mutation plan reserves before it commits); routes nothing. + /// Room for `additional` more entries (a mutation plan reserves before it commits); routes nothing. A map with + /// room already is not copied out of a published snapshot. pub fn try_reserve( &mut self, additional: usize, ) -> Result<(), std::collections::TryReserveError> { - self.0.try_reserve(additional) + if self.0.capacity().saturating_sub(self.0.len()) >= additional { + return Ok(()); + } + Arc::make_mut(&mut self.0).try_reserve(additional) + } +} + +impl RouteTable { + /// The map as it is now, shared with no copy (a [`RouteSnapshot`]). + pub(crate) fn shared(&self) -> Arc> { + Arc::clone(&self.0) + } + + fn shares(&self, map: &Arc>) -> bool { + Arc::ptr_eq(&self.0, map) + } +} + +/// The session's routing tables as of the last finished change of the session state: what `Mux::surface` reads, with +/// no lock. The session state's guard publishes it when it drops after a change (`signaled_mutex::StateGuard`), so +/// a reader sees each change once its guard is released, as a reader that waited for the lock would. +#[derive(Default)] +pub(crate) struct RouteSnapshot { + surfaces: Arc>>, + catalog: Arc>>, + by_runtime: Arc>, +} + +impl RouteSnapshot { + /// The tables of `state`, shared with no copy. + pub(crate) fn of(state: &crate::State) -> Self { + Self { + surfaces: state.surfaces.shared(), + catalog: state.terminal_catalog.shared(), + by_runtime: state.terminal_catalog_by_runtime.shared(), + } + } + + /// Whether `state` still has the tables of this snapshot (no route changed since it was taken). + pub(crate) fn is_current(&self, state: &crate::State) -> bool { + state.surfaces.shares(&self.surfaces) + && state.terminal_catalog.shares(&self.catalog) + && state.terminal_catalog_by_runtime.shares(&self.by_runtime) + } + + /// The surface `id` routes to: its placement, else the catalog owner of the runtime `id` names (the lookup of + /// `Mux::surface`). + pub(crate) fn surface(&self, id: SurfaceId) -> Option> { + self.surfaces + .get(&id) + .or_else(|| self.by_runtime.get(&id).and_then(|terminal| self.catalog.get(terminal))) + .cloned() } } diff --git a/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs b/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs index 84a73dff3ae6..ce7d8115bcb2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs @@ -343,17 +343,9 @@ pub(super) struct ConnectionFacts { pub(super) remote_device: bool, /// The connection's base actor (`ConnectionOrigin::actor`). pub(super) actor: crate::workspace_registry::Actor, - /// The terminal the connection's keys to each attached surface go to (`AttachedSurface::input`). - pub(super) inputs: Vec, -} - -/// `AttachedSurface::input` of one attached surface, with whether the connection can type into it. -pub(super) struct KeptInput { - pub(super) surface: SurfaceId, - pub(super) terminal: Weak, - /// The route generation of `surface` when the terminal was kept (`route_table`). - pub(super) generation: u64, - pub(super) can_type: bool, + /// The surfaces the connection is attached to, each with whether it can type into it + /// (`AttachedSurface::can_type`): its keys there are the person's (server/input_qos.rs). + pub(super) attached: Vec<(SurfaceId, bool)>, } impl ClientFacts { @@ -366,21 +358,16 @@ impl ClientFacts { .clients .iter() .map(|(client, record)| { - let inputs = record + let attached = record .attached .iter() - .map(|(surface, attached)| KeptInput { - surface: *surface, - terminal: attached.input.0.clone(), - generation: attached.input.1, - can_type: attached.can_type(), - }) + .map(|(surface, attached)| (*surface, attached.can_type())) .collect(); let facts = ConnectionFacts { transport: record.transport, remote_device: record.remote_device, actor: record.origin.actor(), - inputs, + attached, }; (*client, facts) }) @@ -423,6 +410,9 @@ pub(crate) struct ClientRegistry { /// Every connection's record. A change publishes the [`ClientFacts`] projection, which a connection's /// own line and key paths read without this lock (R-DAEMON-STATE stage S1). pub(super) state: crate::published::Published, + /// R-DAEMON-STATE stage S1 (`CMUX_TUI_RDS_INPUT`): the line and key paths read [`ClientFacts`] and the + /// session's published routes instead of taking the registry and session locks. + pub(crate) input_lane: crate::rds_switch::InputLaneSwitch, /// Who created each live incognito workspace (incognito_orphans.rs). pub(super) incognito_creators: super::incognito_orphans::IncognitoCreators, } @@ -463,7 +453,9 @@ fn validate_resource_client_label( } impl ClientRegistry { - pub(crate) fn new() -> Self { + /// `input_lane` is the mux's stage S1 switch (R-DAEMON-STATE): on, the registry publishes [`ClientFacts`]. + pub(crate) fn new(input_lane: crate::rds_switch::InputLaneSwitch) -> Self { + let publishing = input_lane.on(); Self { detach_waker: Mutex::new(None), client_presence_observers: Mutex::new(Vec::new()), @@ -494,7 +486,9 @@ impl ClientRegistry { state: crate::published::Published::new( ClientRegistryState::default(), ClientFacts::of, + publishing, ), + input_lane, incognito_creators: Default::default(), } } @@ -567,15 +561,31 @@ impl ClientRegistry { self.state.lock().recover_poison().daemon_handoff.is_some() } + /// Turns stage S1 on or off for this registry (tests): the switch, and the publication it reads. + #[cfg(test)] + pub(crate) fn set_input_lane_for_test(&self, on: bool) { + self.state.set_publishing_for_test(on); + self.input_lane.set_for_test(on); + } + + /// Whether `client` is attached to `surface` and can type into it (`AttachedSurface::can_type`): its keys there + /// are the person's. Reads the published facts, never the registry lock (R-DAEMON-STATE stage S1). + pub(super) fn types_into(&self, client: u64, surface: SurfaceId) -> bool { + let facts = self.state.load().recover_poison(); + facts.clients.get(&client).is_some_and(|connection| { + connection.attached.iter().any(|(attached, can_type)| *attached == surface && *can_type) + }) + } + pub(crate) fn daemon_handoff_in_progress(&self) -> bool { - if crate::rds_switch::input_lane() { + if self.input_lane.on() { return self.state.load().recover_poison().handoff.is_some(); } self.state.lock().recover_poison().daemon_handoff.is_some() } pub(crate) fn daemon_handoff_committed(&self) -> bool { - if crate::rds_switch::input_lane() { + if self.input_lane.on() { return self.state.load().recover_poison().handoff == Some(true); } matches!( diff --git a/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs b/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs index fe7aab886d9b..5775c76c244a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/client_registry_views.rs @@ -395,18 +395,6 @@ impl ClientRegistry { client: u64, surface: SurfaceId, ) -> Option<(Arc, bool)> { - if crate::rds_switch::input_lane() { - // The registry's published facts: no lock that a connect, attach or detach of any client holds. - let facts = self.state.load().recover_poison(); - let kept = - facts.clients.get(&client)?.inputs.iter().find(|kept| kept.surface == surface)?; - if kept.generation != crate::route_table::route_generation(surface) { - return None; - } - let (input, can_type) = (kept.terminal.upgrade()?, kept.can_type); - drop(facts); - return (!input.is_dead()).then_some((input, can_type)); - } let state = self.state.lock().recover_poison(); let attached = state.clients.get(&client)?.attached.get(&surface)?; let (kept, generation) = &attached.input; diff --git a/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs b/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs index b33cac539745..40367569e9d2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs @@ -132,6 +132,16 @@ fn input_surface( writer: &MessageWriter, id: SurfaceId, ) -> anyhow::Result> { + if mux.control_clients.input_lane.on() { + // R-DAEMON-STATE stage S1: the session's published routes give the terminal `id` routes to now (a respawn, + // close or move included) and the client registry's published facts say whether the client types into it. + // Neither takes a lock, so nothing is kept and nothing has to be invalidated. + let surface = get_surface(mux, id)?; + if mux.control_clients.types_into(client, id) { + writer.input_lane.mark_person_input(); + } + return Ok(surface); + } if let Some((surface, can_type)) = mux.control_clients.attached_input_surface(client, id) { if can_type { writer.input_lane.mark_person_input(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs b/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs index af729fcb04fe..5f6d029c1026 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/incognito_remote.rs @@ -181,7 +181,7 @@ pub(super) struct Hidden { /// Whether `client` is a remote device's connection. pub(super) fn is_remote_device(mux: &Mux, client: u64) -> bool { - if crate::rds_switch::input_lane() { + if mux.control_clients.input_lane.on() { // The published facts, never the registry lock (R-DAEMON-STATE S1). let facts = mux.control_clients.state.load().recover_poison(); return facts.clients.get(&client).is_some_and(|connection| connection.remote_device); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs b/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs index 81a63ab16bc4..9553130fd7a6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/origin_gate.rs @@ -220,7 +220,7 @@ fn check_pairing_accept( /// The actor of a durable mutation that `client` asks for on the legacy /// control protocol. A client with no record is never the local user. pub(super) fn connection_actor(mux: &Mux, client: u64) -> Actor { - if crate::rds_switch::input_lane() { + if mux.control_clients.input_lane.on() { // The published facts, never the registry lock (R-DAEMON-STATE S1). let facts = mux.control_clients.state.load().recover_poison(); let Some(connection) = facts.clients.get(&client) else { diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs index 75546f05460e..7147beda6342 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs @@ -106,7 +106,7 @@ impl super::ClientRegistry { /// A poisoned registry lock also gives `None`, which every caller treats /// as remote (fail closed). pub(super) fn transport_of(&self, client: u64) -> Option { - if crate::rds_switch::input_lane() { + if self.input_lane.on() { // The published facts, never the registry lock (R-DAEMON-STATE S1); a poisoned registry still // fails closed. return self.state.load().ok()?.clients.get(&client).map(|facts| facts.transport); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index 7b3e26e52f70..d414c662004f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -54,6 +54,7 @@ struct TypingConnection { impl TypingConnection { fn open() -> Self { let mux = test_mux(); + mux.set_input_lane_for_test(true); let terminal = mux.new_workspace(Some("typing".into()), Some((80, 24))).unwrap(); let (sent, input) = mpsc::channel(); terminal.replace_input_writer_for_test(Box::new(TimedInput(sent))); @@ -146,7 +147,6 @@ fn hold_workspace_registry(mux: &Arc, hold: &dyn Fn()) { /// Sends two keys while another thread holds each lock in turn, and fails when a key waited for one of them. fn assert_keys_wait_for_none(locks: &[(&str, Holder)]) { - crate::rds_switch::set_input_lane_for_test(true); let connection = TypingConnection::open(); // The first key of a view may take the long way; every key after it must not wait. connection.submit("warm", b'w'); @@ -221,7 +221,6 @@ fn a_key_waits_for_neither_the_session_nor_the_workspace_registry_lock() { /// shell, never the replaced one (the route rule of #31940, on the stage's path). #[test] fn after_a_respawn_the_next_key_reaches_the_new_terminal_on_the_input_lane() { - crate::rds_switch::set_input_lane_for_test(true); let connection = TypingConnection::open(); connection.submit("before", b'a'); assert!(connection.arrival(b'a').is_some(), "the first key never reached its terminal"); From 5d0e4a0e2c1c0abfb15f70c0e1588b75d2a811e1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:32:50 +0000 Subject: [PATCH 05/42] test(cmux-tui-core): a key waits for no surface admission lock, with an old-path control (R-DAEMON-STATE S1 step 3 red) Every request of every connection reserves its retained bytes from one daemon-wide budget (ServerSurfaceOperationAdmission) under one mutex before it is queued, and releases them under it after it ran. The S1 census measured a key waiting 300.3 ms behind a 300 ms hold of that lock (hosted run 41648095321). The lane test now has a control: the same two keys with stage S1 off must wait for the held lock, so a lane result can never come from a hold that does not reach the key path (a vacuous green). The control is added to the client registry test too. A test-only hook (cfg(test)) holds the admission lock. Red here: with the lane on, the key still reserves through the mutex. --- .../src/server/tests/rds_input_lane.rs | 103 +++++++++++++----- .../src/server/worker_admission.rs | 8 ++ 2 files changed, 81 insertions(+), 30 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index d414c662004f..b738ada1ab29 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -52,9 +52,15 @@ struct TypingConnection { } impl TypingConnection { + /// A typing connection with stage S1 on (the lane under test). fn open() -> Self { + Self::open_with(true) + } + + /// A typing connection with stage S1 `lane` on or off (off: the old path, a test's control). + fn open_with(lane: bool) -> Self { let mux = test_mux(); - mux.set_input_lane_for_test(true); + mux.set_input_lane_for_test(lane); let terminal = mux.new_workspace(Some("typing".into()), Some((80, 24))).unwrap(); let (sent, input) = mpsc::channel(); terminal.replace_input_writer_for_test(Box::new(TimedInput(sent))); @@ -140,46 +146,59 @@ fn hold_session(mux: &Arc, hold: &dyn Fn()) { mux.with_state(|_| hold()); } +fn hold_surface_admission(mux: &Arc, hold: &dyn Fn()) { + mux.surface_operation_admission.hold_for_test(hold); +} + fn hold_workspace_registry(mux: &Arc, hold: &dyn Fn()) { let _held = mux.workspace_registry.lock().unwrap_or_else(|e| e.into_inner()); hold(); } -/// Sends two keys while another thread holds each lock in turn, and fails when a key waited for one of them. -fn assert_keys_wait_for_none(locks: &[(&str, Holder)]) { - let connection = TypingConnection::open(); - // The first key of a view may take the long way; every key after it must not wait. +/// How long two keys took to reach their PTY while another thread held one lock (the later key's arrival from the +/// first key's submit; `Duration::MAX` when a key never arrived), on a connection that `connection` is. +fn key_wait_while_held(connection: &TypingConnection, index: usize, holder: Holder) -> Duration { + let first = b'a' + u8::try_from(index * 2).unwrap_or(0); + let second = first + 1; + let (held, holding) = mpsc::channel(); + let (release, released) = mpsc::channel::<()>(); + let mux = connection.mux.clone(); + let other = std::thread::spawn(move || { + holder(&mux, &|| { + let _ = held.send(()); + let _ = released.recv_timeout(HOLD); + }); + }); + holding.recv().expect("the other thread took its lock"); + let started = Instant::now(); + let (first_id, second_id) = (format!("{index}-first"), format!("{index}-second")); + connection.submit(&first_id, first); + connection.submit(&second_id, second); + let first_at = connection.arrival(first); + let second_at = connection.arrival(second); + let _ = release.send(()); + other.join().expect("the lock holder ended"); + connection.settle(&[first_id.as_str(), second_id.as_str()]); + match (first_at, second_at) { + (Some(first_at), Some(second_at)) => first_at.max(second_at) - started, + _ => Duration::MAX, + } +} + +/// A connection whose first key took the long way already: every key after it must not wait. +fn warmed(connection: TypingConnection) -> TypingConnection { connection.submit("warm", b'w'); assert!(connection.arrival(b'w').is_some(), "the first key never reached its terminal"); connection.settle(&["warm"]); + connection +} +/// Sends two keys while another thread holds each lock in turn, and fails when a key waited for one of them. +fn assert_keys_wait_for_none(locks: &[(&str, Holder)]) { + let connection = warmed(TypingConnection::open()); let mut waits = Vec::new(); for (index, (name, holder)) in locks.iter().copied().enumerate() { - let first = b'a' + u8::try_from(index * 2).unwrap_or(0); - let second = first + 1; - let (held, holding) = mpsc::channel(); - let (release, released) = mpsc::channel::<()>(); - let mux = connection.mux.clone(); - let other = std::thread::spawn(move || { - holder(&mux, &|| { - let _ = held.send(()); - let _ = released.recv_timeout(HOLD); - }); - }); - holding.recv().expect("the other thread took its lock"); - let started = Instant::now(); - let (first_id, second_id) = (format!("{index}-first"), format!("{index}-second")); - connection.submit(&first_id, first); - connection.submit(&second_id, second); - let first_at = connection.arrival(first); - let second_at = connection.arrival(second); - let _ = release.send(()); - other.join().expect("the lock holder ended"); - connection.settle(&[first_id.as_str(), second_id.as_str()]); - let waited = match (first_at, second_at) { - (Some(first_at), Some(second_at)) => first_at.max(second_at) - started, - _ => Duration::MAX, - }; + let waited = key_wait_while_held(&connection, index, holder); eprintln!("SHARED-LOCKS key wait while {name} is held {HOLD:?}: {waited:?}"); waits.push((name, waited)); } @@ -198,16 +217,40 @@ fn assert_keys_wait_for_none(locks: &[(&str, Holder)]) { ); } +/// The control of a lock test: on the old path (stage S1 off) the key does wait for the held lock, so the hold is +/// real and the lane's result is not a test that holds nothing. +fn assert_old_path_waits_for(name: &str, holder: Holder) { + let connection = warmed(TypingConnection::open_with(false)); + let waited = key_wait_while_held(&connection, 0, holder); + connection.close(); + eprintln!("SHARED-LOCKS control (lane off) key wait while {name} is held {HOLD:?}: {waited:?}"); + assert!( + waited >= BUDGET, + "control: with stage S1 off the key took {waited:?} while {name} was held {HOLD:?}; the hold does not reach \ + the old path, so the lane's test proves nothing" + ); +} + /// Stage S1 step 1: the key line reads the facts of the session's connections from the client registry's published /// projection, never through its lock. #[test] fn a_key_waits_for_no_client_registry_lock() { + assert_old_path_waits_for("client registry (ClientRegistry.state)", hold_client_registry); assert_keys_wait_for_none(&[( "client registry (ClientRegistry.state)", hold_client_registry as Holder, )]); } +/// Stage S1 step 3: every request of every connection reserves its bytes from one daemon-wide budget; the reservation +/// is an atomic compare-and-swap, so a key never waits for another connection's request to reserve or release. +#[test] +fn a_key_waits_for_no_surface_admission_lock() { + let name = "surface admission (ServerSurfaceOperationAdmission)"; + assert_old_path_waits_for(name, hold_surface_admission); + assert_keys_wait_for_none(&[(name, hold_surface_admission as Holder)]); +} + /// #31940 and TYPING FIX 2 took these two off the key; this keeps them off on the stage's new path. #[test] fn a_key_waits_for_neither_the_session_nor_the_workspace_registry_lock() { diff --git a/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs b/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs index 997671854900..fb9104d73eec 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs @@ -148,6 +148,14 @@ impl Drop for ServerSurfaceBytesPermit { } impl ServerSurfaceOperationAdmission { + /// Runs `hold` while this thread holds the admission state lock (test hook: a person's key must not wait for it, + /// R-DAEMON-STATE stage S1). + #[cfg(test)] + pub(super) fn hold_for_test(&self, hold: impl FnOnce()) { + let _state = self.state.lock().recover_poison(); + hold(); + } + pub(super) fn try_reserve_worker(self: &Arc) -> Option { let mut state = self.state.lock().recover_poison(); if state.workers >= SERVER_SURFACE_WORKER_CAPACITY { From 8e5c8a21191de2f7c594631d172687ae6133e638 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:34:00 +0000 Subject: [PATCH 06/42] fix(cmux-tui-core): the surface admission budget reserves with atomics on the input lane (R-DAEMON-STATE S1 step 3) Every request of every connection reserves its retained bytes (and every clear-history its worker) from one daemon-wide budget before it is queued and releases after it ran. Under one mutex, a key waited for any other connection's reservation or release (red at 5d0e4a0e2c1: 300 ms behind a 300 ms hold). With CMUX_TUI_RDS_INPUT=1 the two counters are atomics with compare-and-swap reservations (fetch_update: add only while the sum stays within the capacity; release saturates at zero), so nothing waits. The limits and the refusals are the same as the mutex arm's: a worker is refused at 16 reserved, bytes when the sum would pass 16 MiB. Each permit releases to the arm it reserved from. With the switch off the mutex arm runs as before; it goes with the switch when the stage passes (S1.8). The arm is fixed when the mux is built (ServerSurfaceOperationAdmission::new(publish_routes)); tests set it with Mux::set_input_lane_for_test before any request. The two scheduler tests that counted reserved workers through the mutex read reserved_workers() (both arms). --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 + .../crates/cmux-tui-core/src/mux/construct.rs | 2 +- .../server/tests/outbound_and_scheduler.rs | 4 +- .../src/server/worker_admission.rs | 87 +++++++++++++++++-- 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 841d403d5a0a..52a0fac8e89c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -956,6 +956,7 @@ impl Mux { pub(crate) fn set_input_lane_for_test(&self, on: bool) { self.state.set_route_publishing_for_test(on); self.control_clients.set_input_lane_for_test(on); + self.surface_operation_admission.set_lock_free_for_test(on); } #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs index b0d7dcfb362b..4dee8a3387af 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs @@ -437,7 +437,7 @@ impl Mux { image_pastes: crate::image_paste::ImagePasteStore::default(), history: crate::history_ops::HistoryHost::default(), surface_operation_admission: Arc::new( - crate::server::ServerSurfaceOperationAdmission::default(), + crate::server::ServerSurfaceOperationAdmission::new(publish_routes), ), pairing: PairingBroker::new(), #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs index 86bfcbca410b..b43e332056b5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs @@ -791,7 +791,7 @@ fn queued_same_surface_clears_do_not_reserve_worker_permits() { assert_eq!(scheduler.dispatch(mux.clone(), 0, &mut clear, 0, writer.clone()), Some(true)); } - let reserved_workers = admission.state.lock().unwrap().workers; + let reserved_workers = admission.reserved_workers(); let _ = scheduler.close_and_wait(Duration::from_secs(1)); mux.close_surface(surface.id).unwrap(); @@ -844,7 +844,7 @@ fn queued_wait_releases_clear_worker_permit_after_clear_settles() { ); } drop(state); - let active_clear_workers = admission.state.lock().unwrap().workers; + let active_clear_workers = admission.reserved_workers(); let drained = scheduler.close_and_wait(Duration::from_secs(1)); mux.close_surface(surface.id).unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs b/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs index fb9104d73eec..f34fc056c30e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs @@ -9,6 +9,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::sync::Condvar; use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; #[cfg(test)] use std::time::Instant; @@ -114,9 +115,21 @@ pub(super) struct ServerSurfaceOperationState { retained_bytes: usize, } +/// Daemon-wide admission of surface operations: clear-history workers by count and queued requests by retained bytes. +/// Every request of every connection reserves here before it is queued and releases after it ran. +/// +/// R-DAEMON-STATE stage S1 (`CMUX_TUI_RDS_INPUT`): with the lane on, the counters are atomics with compare-and-swap +/// reservations, so no request, and no key of the person, waits for another connection's reservation. With it off, +/// the old mutex counts (deleted with the switch when the stage passes). #[derive(Default)] pub(crate) struct ServerSurfaceOperationAdmission { + /// The old arm's counters (stage S1 off). pub(super) state: Mutex, + /// The lane's counters (stage S1 on). + workers: AtomicUsize, + retained_bytes: AtomicUsize, + /// Which arm reserves; fixed when the mux is built (`Mux::set_input_lane_for_test` sets it before any request). + lock_free: AtomicBool, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -126,29 +139,74 @@ pub(super) enum ServerSurfaceAdmissionError { pub(super) struct ServerSurfaceWorkerPermit { admission: Arc, + /// The arm it reserved from; it releases to the same one. + lock_free: bool, } impl Drop for ServerSurfaceWorkerPermit { fn drop(&mut self) { - let mut state = self.admission.state.lock().recover_poison(); - state.workers = state.workers.saturating_sub(1); + if self.lock_free { + release(&self.admission.workers, 1); + } else { + let mut state = self.admission.state.lock().recover_poison(); + state.workers = state.workers.saturating_sub(1); + } } } pub(super) struct ServerSurfaceBytesPermit { pub(super) admission: Arc, pub(super) retained_bytes: usize, + /// The arm it reserved from; it releases to the same one. + lock_free: bool, } impl Drop for ServerSurfaceBytesPermit { fn drop(&mut self) { - let mut state = self.admission.state.lock().recover_poison(); - state.retained_bytes = state.retained_bytes.saturating_sub(self.retained_bytes); + if self.lock_free { + release(&self.admission.retained_bytes, self.retained_bytes); + } else { + let mut state = self.admission.state.lock().recover_poison(); + state.retained_bytes = state.retained_bytes.saturating_sub(self.retained_bytes); + } } } +/// Adds `amount` to `counter` when the sum stays within `capacity`; false (and nothing added) otherwise. +fn reserve(counter: &AtomicUsize, amount: usize, capacity: usize) -> bool { + counter + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |held| { + held.checked_add(amount).filter(|total| *total <= capacity) + }) + .is_ok() +} + +/// Subtracts what a permit reserved; a release never goes below zero. +fn release(counter: &AtomicUsize, amount: usize) { + let _ = counter.fetch_update(Ordering::AcqRel, Ordering::Acquire, |held| { + Some(held.saturating_sub(amount)) + }); +} + impl ServerSurfaceOperationAdmission { - /// Runs `hold` while this thread holds the admission state lock (test hook: a person's key must not wait for it, + /// The admission of a mux whose stage S1 switch is `lock_free` (R-DAEMON-STATE). + pub(crate) fn new(lock_free: bool) -> Self { + Self { lock_free: AtomicBool::new(lock_free), ..Self::default() } + } + + /// Turns stage S1 on or off (tests, before any request reserves). + #[cfg(test)] + pub(crate) fn set_lock_free_for_test(&self, on: bool) { + self.lock_free.store(on, Ordering::Relaxed); + } + + /// The clear-history workers reserved now, on either arm (tests). + #[cfg(test)] + pub(super) fn reserved_workers(&self) -> usize { + self.workers.load(Ordering::Acquire) + self.state.lock().recover_poison().workers + } + + /// Runs `hold` while this thread holds the old arm's lock (test hook: a person's key must not wait for it, /// R-DAEMON-STATE stage S1). #[cfg(test)] pub(super) fn hold_for_test(&self, hold: impl FnOnce()) { @@ -157,18 +215,33 @@ impl ServerSurfaceOperationAdmission { } pub(super) fn try_reserve_worker(self: &Arc) -> Option { + if self.lock_free.load(Ordering::Relaxed) { + return reserve(&self.workers, 1, SERVER_SURFACE_WORKER_CAPACITY) + .then(|| ServerSurfaceWorkerPermit { admission: self.clone(), lock_free: true }); + } let mut state = self.state.lock().recover_poison(); if state.workers >= SERVER_SURFACE_WORKER_CAPACITY { return None; } state.workers += 1; - Some(ServerSurfaceWorkerPermit { admission: self.clone() }) + Some(ServerSurfaceWorkerPermit { admission: self.clone(), lock_free: false }) } pub(super) fn try_reserve_bytes( self: &Arc, retained_bytes: usize, ) -> Result { + if self.lock_free.load(Ordering::Relaxed) { + if !reserve(&self.retained_bytes, retained_bytes, SERVER_SURFACE_RETAINED_BYTE_CAPACITY) + { + return Err(ServerSurfaceAdmissionError::RetainedByteCapacity); + } + return Ok(ServerSurfaceBytesPermit { + admission: self.clone(), + retained_bytes, + lock_free: true, + }); + } let mut state = self.state.lock().recover_poison(); if retained_bytes > SERVER_SURFACE_RETAINED_BYTE_CAPACITY.saturating_sub(state.retained_bytes) @@ -176,6 +249,6 @@ impl ServerSurfaceOperationAdmission { return Err(ServerSurfaceAdmissionError::RetainedByteCapacity); } state.retained_bytes += retained_bytes; - Ok(ServerSurfaceBytesPermit { admission: self.clone(), retained_bytes }) + Ok(ServerSurfaceBytesPermit { admission: self.clone(), retained_bytes, lock_free: false }) } } From 154730a175df7dece87ebd3b849b40cecb939d83 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:08:05 +0000 Subject: [PATCH 07/42] test(cmux-tui-core): a key waits for neither the link peers nor the proxy authors lock (R-DAEMON-STATE S1 step 2 red) Two more shared locks on the person's key line (DESIGN.md Appendix A.2 steps 3, 7, 12 and 13): the link peers map (RemoteRelayState.peers: the remote-client check and the request's actor) and the host proxy authors (AgentSessions.proxy_authors: the proxy-link check on the reader thread). The S1 census measured a full 300 ms wait on each (hosted run 41648095321: 300.5 ms and 300.4 ms). The test holds each lock 300 ms on another thread while two keys go through the real line path. It first proves the hold reaches the old path (stage switch off: the key waits), then expects the stage's path not to wait. Red on this commit: the stage does not publish either value yet. --- .../src/server/agent_session_attach.rs | 7 ++++++ .../src/server/tests/rds_input_lane.rs | 25 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs index 96932c47d39a..5c7e99224aea 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs @@ -181,6 +181,13 @@ impl AgentSessions { self.spoken.lock().unwrap_or_else(|e| e.into_inner()).contains(&client) } + /// Runs `hold` while this thread holds the host proxy authors (tests that prove a key does not wait for them). + #[cfg(test)] + pub(crate) fn hold_proxy_authors_for_test(&self, hold: &dyn Fn()) { + let _held = self.proxy_authors.lock().unwrap_or_else(|e| e.into_inner()); + hold(); + } + /// The connection ended: every attachment of it closes. pub(crate) fn disconnect(&self, client: u64) { self.inventory_watches.disconnect(client); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index b738ada1ab29..3b3a0ea10d70 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -155,6 +155,15 @@ fn hold_workspace_registry(mux: &Arc, hold: &dyn Fn()) { hold(); } +fn hold_link_peers(mux: &Arc, hold: &dyn Fn()) { + let _held = mux.remote_relay().peers.lock().unwrap_or_else(|e| e.into_inner()); + hold(); +} + +fn hold_proxy_authors(mux: &Arc, hold: &dyn Fn()) { + mux.control_clients.agent_sessions.hold_proxy_authors_for_test(hold); +} + /// How long two keys took to reach their PTY while another thread held one lock (the later key's arrival from the /// first key's submit; `Duration::MAX` when a key never arrived), on a connection that `connection` is. fn key_wait_while_held(connection: &TypingConnection, index: usize, holder: Holder) -> Duration { @@ -251,6 +260,22 @@ fn a_key_waits_for_no_surface_admission_lock() { assert_keys_wait_for_none(&[(name, hold_surface_admission as Holder)]); } +/// Stage S1 step 2: the key line asks whether its connection has a link peer record (the remote-client check and the +/// request's actor) and whether it is a host proxy's link for a teammate. It reads both from published values, never +/// through the link peers lock (every remote bind, disconnect and revocation sweep holds it) or the proxy authors lock +/// (every proxy bind and every client's disconnect holds it). +#[test] +fn a_key_waits_for_neither_the_link_peers_nor_the_proxy_authors_lock() { + let peers = "link peers (RemoteRelayState.peers)"; + let authors = "host proxy authors (AgentSessions.proxy_authors)"; + assert_old_path_waits_for(peers, hold_link_peers); + assert_old_path_waits_for(authors, hold_proxy_authors); + assert_keys_wait_for_none(&[ + (peers, hold_link_peers as Holder), + (authors, hold_proxy_authors as Holder), + ]); +} + /// #31940 and TYPING FIX 2 took these two off the key; this keeps them off on the stage's new path. #[test] fn a_key_waits_for_neither_the_session_nor_the_workspace_registry_lock() { From 48c4d9e949ca3d315115d47b2be2858d52de60de Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:34:46 +0000 Subject: [PATCH 08/42] test(cmux-tui-core): a change cut short by a panic is never published (R-DAEMON-STATE S1 red) A published value's guard publishes its projection in Drop::drop, which also runs while a panic unwinds, before its MutexGuard field drops and marks the mutex poisoned. A writer that panics in the middle of a change so publishes the half-made value, and in the moment between the publication and the poisoning a reader gets it as a good value: a fail-open window for readers that fail closed on a poisoned value (transport_of, peer_checked). Red on this commit: the test sees the half-made value 9 in the poisoned projection. --- cmux-tui/crates/cmux-tui-core/src/published.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/published.rs b/cmux-tui/crates/cmux-tui-core/src/published.rs index 0bbc9e386a4a..66cc06b613f9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/published.rs +++ b/cmux-tui/crates/cmux-tui-core/src/published.rs @@ -191,6 +191,23 @@ mod tests { assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 1); } + /// A writer that panics in the middle of a change leaves a half-made value: its guard still drops (the unwind + /// runs it), but it publishes nothing, so no reader ever sees the half-made change, not even in the moment before + /// the mutex is marked poisoned. Readers then fail closed with the last finished projection. + #[test] + fn a_change_cut_short_by_a_panic_is_never_published() { + let published = Published::new(Counter::default(), project, true); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut guard = published.lock().unwrap_or_else(|poison| poison.into_inner()); + guard.value = 9; + panic!("a writer panics in the middle of a change"); + })); + let Err(poisoned) = published.load() else { + panic!("a reader of a poisoned value must be able to fail closed"); + }; + assert_eq!(**poisoned.into_inner(), 0, "the half-made change was published"); + } + #[test] fn a_poisoned_value_gives_its_projection_in_the_error() { let published = Published::new(Counter::default(), project, true); From 44623898e7fb3e6a5e29a1558ea6a89ccdbabd67 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:53:23 +0000 Subject: [PATCH 09/42] test(cmux-tui-core): a reservation unshares a published route map; a panicked change publishes no routes (R-DAEMON-STATE S1 red) Two defects the #34014 review found on the input lane (both only with CMUX_TUI_RDS_INPUT on): - RouteTable::try_reserve returns early when the map has room, also when a published route snapshot shares the map. The insert after the durable commit then copies the whole table (Arc::make_mut), an allocation that ResourceMutationPlan::apply forbids ("Plan builders reserve all needed capacities before returning"). - The session state guard publishes the routes of a change that a panic cut short: Drop::drop runs during the unwind, before the MutexGuard field poisons the mutex, so the lane routes by half-made tables. Red on this commit: the insert changes the map's pointer, and the routes snapshot is replaced. --- .../src/mux/tests/signaled_mutex.rs | 19 +++++++++++++ .../crates/cmux-tui-core/src/route_table.rs | 28 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs b/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs index aa00c7450f5d..47daeb6cde6c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs @@ -107,3 +107,22 @@ fn lock_detail_reports_per_site_hold_and_wait_percentiles() { assert!(long.hold_p99_us >= 10_000 && long.hold_p99_us <= long.hold_max_us, "{long:?}"); assert!(long.hold_max_us >= 20_000, "{long:?}"); } + +/// A writer that panics in the middle of a change leaves half-made routing tables. They are never published (the +/// guard drops during the unwind, then the mutex is poisoned), so the input lane keeps routing by the last finished +/// change (R-DAEMON-STATE stage S1). +#[test] +fn routes_of_a_change_cut_short_by_a_panic_are_never_published() { + let state = test_mux().with_state(State::clone); + let mutex = super::super::signaled_mutex::StateMutex::new(state, true); + let before = arc_swap::Guard::into_inner(mutex.routes()); + let terminal = + crate::resource::TerminalPublicId::parse("term_00000000000000000000000000000077").unwrap(); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut state = mutex.lock().unwrap_or_else(PoisonError::into_inner); + state.terminal_catalog_by_runtime.insert(9_999_999, terminal); + panic!("a writer panics in the middle of a change"); + })); + let after = arc_swap::Guard::into_inner(mutex.routes()); + assert!(Arc::ptr_eq(&before, &after), "the routes of the half-made change were published"); +} diff --git a/cmux-tui/crates/cmux-tui-core/src/route_table.rs b/cmux-tui/crates/cmux-tui-core/src/route_table.rs index 9e527eb372bb..32d6d70cee50 100644 --- a/cmux-tui/crates/cmux-tui-core/src/route_table.rs +++ b/cmux-tui/crates/cmux-tui-core/src/route_table.rs @@ -210,3 +210,31 @@ impl<'a, K, V> IntoIterator for &'a RouteTable { self.0.iter() } } + +#[cfg(test)] +mod tests { + use super::{RouteKey, RouteTable, SurfaceId}; + use std::sync::Arc; + + /// A key that routes no surface (no route generation moves). + impl RouteKey for u32 { + fn routed_id(&self, _: &u32) -> Option { + None + } + } + + /// A mutation plan reserves room before its durable commit and inserts after it, and the insert must allocate + /// nothing (`ResourceMutationPlan::apply`). With stage S1 on, a published route snapshot shares the map: the + /// reservation copies it then, before the commit, so the insert after the commit copies nothing. + #[test] + fn a_reservation_unshares_a_published_map_so_the_insert_after_the_commit_allocates_nothing() { + let mut table = RouteTable::::new(); + table.try_reserve(8).unwrap(); + let published = table.shared(); + table.try_reserve(1).unwrap(); + let reserved = Arc::as_ptr(&table.0); + table.insert(1, 1); + assert_eq!(Arc::as_ptr(&table.0), reserved, "the insert after the commit copied the map"); + assert!(published.is_empty(), "the published snapshot changed"); + } +} From 8f7d40500e8776eac62c9fa30ef932650588c641 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:57:48 +0000 Subject: [PATCH 10/42] fix(cmux-tui-core): link peers and host proxy authors are published; the S1.1 review follow-ups (R-DAEMON-STATE S1 step 2) A person's key line asked two more shared locks (DESIGN.md Appendix A.2 steps 3, 7, 12, 13): the link peers map (the remote-client check and the request's actor) and the host proxy authors (the proxy-link check on the reader thread). Every remote bind, disconnect and revocation sweep holds the first; every proxy bind and every client's disconnect holds the second. - published.rs moves to cmux-tui-util (daemon primitives, no daemon state) so cmux-tui-remote-access can use it. RemoteRelayState.peers becomes a Published copy of the records; peer_checked reads it with no lock when the mux's stage switch is on (Published::publishing), and a poisoned map still fails closed. AgentSessions.proxy_authors publishes the same way; proxy::author reads it. A disconnect of a client without a record publishes nothing. - Review follow-ups of #34014 (review comment 6612546773): - RouteTable::try_reserve copies a map a published snapshot shares, before the durable commit, so the insert after the commit allocates nothing (ResourceMutationPlan::apply). - A change a panic cuts short is never published, by the state guard (routes) or by a PublishGuard: Drop skips the publication when a panic began while the guard held the lock (std's own poisoning test), so no reader sees half-made tables, not even before the mutex is poisoned. - A PublishGuard frees the projection it replaced after the unlock (as the state guard does). - The two poison tests of the relay run on both arms. - ClientFacts and types_into move to server/client_registry/facts.rs (god-file warning). Same switch, CMUX_TUI_RDS_INPUT, default off: with it off nothing is published and every read takes its lock as on the base. --- cmux-tui/Cargo.lock | 2 + cmux-tui/crates/cmux-tui-core/src/lib.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 + .../crates/cmux-tui-core/src/mux/construct.rs | 15 +- .../cmux-tui-core/src/mux/conversations.rs | 14 +- .../cmux-tui-core/src/mux/signaled_mutex.rs | 11 +- .../crates/cmux-tui-core/src/route_table.rs | 9 +- .../src/server/agent_session_attach.rs | 32 +++- .../src/server/agent_session_proxy.rs | 13 +- .../src/server/client_registry.rs | 69 +-------- .../src/server/client_registry/facts.rs | 66 +++++++++ .../src/server/remote_relay/mod.rs | 2 +- .../src/server/remote_relay/revocation.rs | 2 +- .../src/server/remote_relay/tests.rs | 45 +++--- .../crates/cmux-tui-remote-access/Cargo.toml | 1 + .../src/remote_relay_state.rs | 39 ++++- cmux-tui/crates/cmux-tui-util/Cargo.toml | 5 +- cmux-tui/crates/cmux-tui-util/src/lib.rs | 8 +- .../src/published.rs | 137 +++++++++++++----- 19 files changed, 318 insertions(+), 155 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/server/client_registry/facts.rs rename cmux-tui/crates/{cmux-tui-core => cmux-tui-util}/src/published.rs (60%) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 5dfc0d3a7c57..4910d384e491 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -1941,6 +1941,7 @@ dependencies = [ "anyhow", "base64", "cmux-link", + "cmux-tui-util", "getrandom 0.3.4", ] @@ -1952,6 +1953,7 @@ version = "0.0.0" name = "cmux-tui-util" version = "0.1.0" dependencies = [ + "arc-swap", "cmux-tui-platform", "getrandom 0.3.4", "libc", diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index a4a29c417603..0f57f302f644 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -71,7 +71,7 @@ mod pty_input_room; #[cfg(unix)] mod pty_write; /// A mutex-guarded value whose readers load a published projection without the lock (R-DAEMON-STATE). -mod published; +use cmux_tui_util::published; mod terminal_title; use cmux_tui_remote_access::remote_relay_state; mod login_shell; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 52a0fac8e89c..d87524cf570d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -956,6 +956,7 @@ impl Mux { pub(crate) fn set_input_lane_for_test(&self, on: bool) { self.state.set_route_publishing_for_test(on); self.control_clients.set_input_lane_for_test(on); + self.remote_relay().peers.set_publishing_for_test(on); self.surface_operation_admission.set_lock_free_for_test(on); } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs index 4dee8a3387af..a4543472bdfb 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs @@ -202,9 +202,11 @@ impl Mux { let launch_identity = crate::launch_credential::LaunchIdentity::load( registry.session_journal_database_path().as_deref().and_then(Path::parent), ); - // One stage S1 switch for the mux (R-DAEMON-STATE): the state's routes and the client registry's facts. + // One stage S1 switch for the mux (R-DAEMON-STATE): on, the values a person's key reads are published (the + // state's routes, the client registry's facts, the link peers, the host proxy authors) and the surface + // admission budget reserves with atomics. let input_lane = crate::rds_switch::InputLaneSwitch::from_env(); - let publish_routes = input_lane.on(); + let lane_on = input_lane.on(); let mux = Arc::new(Mux { launch_identity, registry_connection: registry.connection.clone(), @@ -216,7 +218,7 @@ impl Mux { registry_identity, resource_projection_stats, started_at: Instant::now(), - state: signaled_mutex::StateMutex::new(state, publish_routes), + state: signaled_mutex::StateMutex::new(state, lane_on), subscribers: MuxEventBroadcaster::default(), config_reload: Mutex::new(ConfigReloadState::default()), config_reload_changed: Condvar::new(), @@ -233,7 +235,10 @@ impl Mux { client_sizing: Mutex::new(ClientSizingState::default()), client_focus_memory: Mutex::new(Vec::new()), last_reported_focus: Mutex::new(None), - conversations: Default::default(), + conversations: crate::conversation_store::ConversationHost { + remote: crate::remote_relay_state::RemoteRelayState::new(lane_on), + ..Default::default() + }, chiefs: Default::default(), cloud: Default::default(), history_search: OnceLock::new(), @@ -437,7 +442,7 @@ impl Mux { image_pastes: crate::image_paste::ImagePasteStore::default(), history: crate::history_ops::HistoryHost::default(), surface_operation_admission: Arc::new( - crate::server::ServerSurfaceOperationAdmission::new(publish_routes), + crate::server::ServerSurfaceOperationAdmission::new(lane_on), ), pairing: PairingBroker::new(), #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/conversations.rs b/cmux-tui/crates/cmux-tui-core/src/mux/conversations.rs index 68b514f0db85..31f910fe5efd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/conversations.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/conversations.rs @@ -371,12 +371,14 @@ impl Mux { } // Safety: a removal never grants access, so a poisoned peers lock // still drops the record. - self.conversations - .remote - .peers - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&client); + { + let mut peers = + self.conversations.remote.peers.lock().unwrap_or_else(PoisonError::into_inner); + // A connection without a record (every local one) changes nothing, so it publishes nothing. + if peers.contains_key(&client) { + peers.remove(&client); + } + } // Its attachment uploads end with it (never opens the store). if let Some(store) = self.conversations.store.lock().unwrap_or_else(PoisonError::into_inner).as_mut() diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs b/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs index fe69bf248b24..cc4a500b2c82 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs @@ -237,6 +237,8 @@ pub(crate) struct StateGuard<'a> { /// The routing snapshot this guard replaced. It drops after `guard` (fields drop in order), so the tables and /// surfaces it may hold last are freed after the lock is released, never under it. retired: Option>, + /// Whether this thread was already unwinding when it took the lock, as `std::sync::MutexGuard` records it. + panicking_at_lock: bool, } impl<'a> StateGuard<'a> { @@ -269,6 +271,7 @@ impl<'a> StateGuard<'a> { acquired_at: Instant::now(), _rank: HeldRank::record(LockRank::MuxState, STATE_LOCK_NAME), retired: None, + panicking_at_lock: std::thread::panicking(), } } } @@ -290,9 +293,13 @@ pub(crate) fn state_locks_taken_on_this_thread() -> u64 { impl Drop for StateGuard<'_> { fn drop(&mut self) { // Publish the routing tables a change replaced, while the lock is still held, so publications follow the - // order of the changes (R-DAEMON-STATE stage S1). A guard that changed no route compares three pointers. + // order of the changes (R-DAEMON-STATE stage S1). A guard that changed no route compares three pointers. A + // panic that began while this guard held the lock cut its change short: the `MutexGuard` field poisons the + // mutex right after this (the same test std makes), and the half-made tables are never published. let owner = self.owner; - if owner.publish_routes.load(std::sync::atomic::Ordering::Relaxed) + let cut_short = std::thread::panicking() && !self.panicking_at_lock; + if !cut_short + && owner.publish_routes.load(std::sync::atomic::Ordering::Relaxed) && !owner.routes.load().is_current(&self.guard) { let published = std::sync::Arc::new(crate::route_table::RouteSnapshot::of(&self.guard)); diff --git a/cmux-tui/crates/cmux-tui-core/src/route_table.rs b/cmux-tui/crates/cmux-tui-core/src/route_table.rs index 32d6d70cee50..b5039b91f657 100644 --- a/cmux-tui/crates/cmux-tui-core/src/route_table.rs +++ b/cmux-tui/crates/cmux-tui-core/src/route_table.rs @@ -133,15 +133,14 @@ impl, V: Clone> RouteTable { Some(removed) } - /// Room for `additional` more entries (a mutation plan reserves before it commits); routes nothing. A map with - /// room already is not copied out of a published snapshot. + /// Room for `additional` more entries; routes nothing. A mutation plan reserves before its durable commit, so + /// the insert after the commit allocates nothing (`ResourceMutationPlan::apply`). A map a published snapshot + /// shares is copied here, before the commit, never by that insert; the plan reserves and applies under one + /// state guard, so no publication shares the map again in between. pub fn try_reserve( &mut self, additional: usize, ) -> Result<(), std::collections::TryReserveError> { - if self.0.capacity().saturating_sub(self.0.len()) >= additional { - return Ok(()); - } Arc::make_mut(&mut self.0).try_reserve(additional) } } diff --git a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs index 5c7e99224aea..650a71effec2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_attach.rs @@ -107,8 +107,10 @@ pub(crate) struct AgentSessions { pub(super) inventory_reads: AtomicUsize, pub(super) inventory_watches: super::agent_inventory::watch::Watches, state: Mutex, - /// WhoIs attestations from the same-user host proxy, fixed for each link. - proxy_authors: Mutex>, + /// WhoIs attestations from the same-user host proxy, fixed for each link. Every line of every connection asks + /// whether it is a proxy's link, so with publishing on (R-DAEMON-STATE stage S1) that read loads the published + /// copy and never waits for a bind or for another client's disconnect. + proxy_authors: crate::published::Published, HashMap>, /// Link shares for guests (`agent-session-share-*`, agent_session_shares.rs). link_shares: proxy::shares::LinkShares, /// Unix connections past their first line: a proxy binds only on its first. @@ -141,6 +143,24 @@ impl AttachState { } impl AgentSessions { + /// The attachments of a daemon whose stage S1 switch (R-DAEMON-STATE) is `publish_proxy_authors`. + pub(crate) fn new(publish_proxy_authors: bool) -> Self { + Self { + proxy_authors: crate::published::Published::new( + HashMap::new(), + HashMap::clone, + publish_proxy_authors, + ), + ..Self::default() + } + } + + /// Turns the stage S1 publication of the proxy authors on or off (tests). + #[cfg(test)] + pub(crate) fn set_publishing_for_test(&self, on: bool) { + self.proxy_authors.set_publishing_for_test(on); + } + /// The live attachment of `surface` on `client`; an ended one is dropped. fn attachment(&self, client: u64, surface: SurfaceId) -> Option> { let mut state = self.lock(); @@ -191,7 +211,13 @@ impl AgentSessions { /// The connection ended: every attachment of it closes. pub(crate) fn disconnect(&self, client: u64) { self.inventory_watches.disconnect(client); - self.proxy_authors.lock().unwrap_or_else(|e| e.into_inner()).remove(&client); + { + let mut authors = self.proxy_authors.lock().unwrap_or_else(|e| e.into_inner()); + // A client that never bound (all but a proxy's links) changes nothing, so it publishes nothing. + if authors.contains_key(&client) { + authors.remove(&client); + } + } self.spoken.lock().unwrap_or_else(|e| e.into_inner()).remove(&client); let gone: Vec> = { let mut state = self.lock(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_proxy.rs b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_proxy.rs index a8462655e237..9998cf370ccc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/agent_session_proxy.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/agent_session_proxy.rs @@ -71,13 +71,12 @@ pub(super) fn scope( } pub(super) fn author(mux: &Mux, client: u64) -> Option { - mux.control_clients - .agent_sessions - .proxy_authors - .lock() - .unwrap_or_else(|e| e.into_inner()) - .get(&client) - .cloned() + let authors = &mux.control_clients.agent_sessions.proxy_authors; + if authors.publishing() { + // The published binds (R-DAEMON-STATE stage S1): every line asks, and none waits for a bind or a disconnect. + return authors.load().unwrap_or_else(|e| e.into_inner()).get(&client).cloned(); + } + authors.lock().unwrap_or_else(|e| e.into_inner()).get(&client).cloned() } pub(super) fn bind( diff --git a/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs b/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs index ce7d8115bcb2..f3e9b002d7ba 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/client_registry.rs @@ -3,9 +3,11 @@ //! surfaces and view leases) and the daemon handoff reservation. Attach and //! view-lease methods are in `client_registry_views`. +mod facts; #[cfg(unix)] pub(super) mod machine_catalog; mod record; +pub(super) use facts::ClientFacts; pub(super) use record::ClientRecord; use super::CREATION_ATTEMPT_KEYS_CAPABILITY; @@ -326,56 +328,6 @@ pub(super) struct ClientRegistryState { pub(super) daemon_handoff: Option, } -/// What a connection's own line and key paths read about the session's connections, published after every change -/// of the client registry and read with one atomic load, never through its lock (R-DAEMON-STATE stage S1, -/// handoff/rewrites/R-DAEMON-STATE/DESIGN.md section 2.6). Every connect, disconnect, attach and detach of every -/// client holds the registry lock; a person's key must not wait for any of them. -pub(super) struct ClientFacts { - /// The daemon handoff, when one is reserved: `true` once it was acknowledged (committed). - pub(super) handoff: Option, - pub(super) clients: HashMap, -} - -/// One connection's facts in [`ClientFacts`]. -pub(super) struct ConnectionFacts { - pub(super) transport: ClientTransport, - /// `ClientRecord::remote_device`. - pub(super) remote_device: bool, - /// The connection's base actor (`ConnectionOrigin::actor`). - pub(super) actor: crate::workspace_registry::Actor, - /// The surfaces the connection is attached to, each with whether it can type into it - /// (`AttachedSurface::can_type`): its keys there are the person's (server/input_qos.rs). - pub(super) attached: Vec<(SurfaceId, bool)>, -} - -impl ClientFacts { - /// The projection of `state` (runs under the registry lock after each change). - fn of(state: &ClientRegistryState) -> Self { - let handoff = state - .daemon_handoff - .map(|reservation| matches!(reservation, DaemonHandoffReservation::Committed(_))); - let clients = state - .clients - .iter() - .map(|(client, record)| { - let attached = record - .attached - .iter() - .map(|(surface, attached)| (*surface, attached.can_type())) - .collect(); - let facts = ConnectionFacts { - transport: record.transport, - remote_device: record.remote_device, - actor: record.origin.actor(), - attached, - }; - (*client, facts) - }) - .collect(); - Self { handoff, clients } - } -} - pub(crate) struct ClientRegistry { /// Called when a surface loses its last attached client (the idle-close /// reaper starts that terminal's unattached period). @@ -467,7 +419,7 @@ impl ClientRegistry { browser_runtimes: Default::default(), size_leases: Default::default(), #[cfg(unix)] - agent_sessions: Default::default(), + agent_sessions: agent_session_attach::AgentSessions::new(publishing), snapshot_viewers: Default::default(), apps: crate::apps::AppsSlot::default(), scripts: crate::scripts::ScriptsSlot::default(), @@ -561,22 +513,17 @@ impl ClientRegistry { self.state.lock().recover_poison().daemon_handoff.is_some() } - /// Turns stage S1 on or off for this registry (tests): the switch, and the publication it reads. + /// Turns stage S1 on or off for this registry (tests): the switch, and the publications it reads (the facts, the + /// proxy authors). Only part of the stage: tests call `Mux::set_input_lane_for_test`, which also publishes the + /// session's routes and the link peers and switches the admission budget. #[cfg(test)] pub(crate) fn set_input_lane_for_test(&self, on: bool) { self.state.set_publishing_for_test(on); + #[cfg(unix)] + self.agent_sessions.set_publishing_for_test(on); self.input_lane.set_for_test(on); } - /// Whether `client` is attached to `surface` and can type into it (`AttachedSurface::can_type`): its keys there - /// are the person's. Reads the published facts, never the registry lock (R-DAEMON-STATE stage S1). - pub(super) fn types_into(&self, client: u64, surface: SurfaceId) -> bool { - let facts = self.state.load().recover_poison(); - facts.clients.get(&client).is_some_and(|connection| { - connection.attached.iter().any(|(attached, can_type)| *attached == surface && *can_type) - }) - } - pub(crate) fn daemon_handoff_in_progress(&self) -> bool { if self.input_lane.on() { return self.state.load().recover_poison().handoff.is_some(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server/client_registry/facts.rs b/cmux-tui/crates/cmux-tui-core/src/server/client_registry/facts.rs new file mode 100644 index 000000000000..1c1bf4340631 --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/server/client_registry/facts.rs @@ -0,0 +1,66 @@ +//! What a connection's own line and key paths read about the session's connections: the client registry's +//! published projection (R-DAEMON-STATE stage S1), and the reads made from it. +use super::{ClientRegistry, ClientRegistryState, ClientTransport, DaemonHandoffReservation}; +use crate::{SurfaceId, panic_free::RecoverPoison}; +use std::collections::HashMap; + +/// What a connection's own line and key paths read about the session's connections, published after every change +/// of the client registry and read with one atomic load, never through its lock (R-DAEMON-STATE stage S1, +/// handoff/rewrites/R-DAEMON-STATE/DESIGN.md section 2.6). Every connect, disconnect, attach and detach of every +/// client holds the registry lock; a person's key must not wait for any of them. +pub(in crate::server) struct ClientFacts { + /// The daemon handoff, when one is reserved: `true` once it was acknowledged (committed). + pub(in crate::server) handoff: Option, + pub(in crate::server) clients: HashMap, +} + +/// One connection's facts in [`ClientFacts`]. +pub(in crate::server) struct ConnectionFacts { + pub(in crate::server) transport: ClientTransport, + /// `ClientRecord::remote_device`. + pub(in crate::server) remote_device: bool, + /// The connection's base actor (`ConnectionOrigin::actor`). + pub(in crate::server) actor: crate::workspace_registry::Actor, + /// The surfaces the connection is attached to, each with whether it can type into it + /// (`AttachedSurface::can_type`): its keys there are the person's (server/input_qos.rs). + pub(in crate::server) attached: Vec<(SurfaceId, bool)>, +} + +impl ClientFacts { + /// The projection of `state` (runs under the registry lock after each change). + pub(super) fn of(state: &ClientRegistryState) -> Self { + let handoff = state + .daemon_handoff + .map(|reservation| matches!(reservation, DaemonHandoffReservation::Committed(_))); + let clients = state + .clients + .iter() + .map(|(client, record)| { + let attached = record + .attached + .iter() + .map(|(surface, attached)| (*surface, attached.can_type())) + .collect(); + let facts = ConnectionFacts { + transport: record.transport, + remote_device: record.remote_device, + actor: record.origin.actor(), + attached, + }; + (*client, facts) + }) + .collect(); + Self { handoff, clients } + } +} + +impl ClientRegistry { + /// Whether `client` is attached to `surface` and can type into it (`AttachedSurface::can_type`): its keys there + /// are the person's. Reads the published facts, never the registry lock; only the input lane calls it. + pub(in crate::server) fn types_into(&self, client: u64, surface: SurfaceId) -> bool { + let facts = self.state.load().recover_poison(); + facts.clients.get(&client).is_some_and(|connection| { + connection.attached.iter().any(|(attached, can_type)| *attached == surface && *can_type) + }) + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs index 7147beda6342..846b4482bbc6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/mod.rs @@ -177,7 +177,7 @@ impl Mux { } else { None }; - match lock_checked(&relay.peers, RelayLock::Peers) { + match relay.peers_checked() { Ok(mut peers) => { peers.insert(client, peer.clone()); if let Some(clients) = cloud_clients.as_deref_mut() { diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/revocation.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/revocation.rs index 0082cd830000..3ef7ea37861a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/revocation.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/revocation.rs @@ -76,7 +76,7 @@ impl Mux { pub fn enforce_remote_limits(self: &Arc) -> Result, RelayStateError> { let collected = (|| { let revocation = lock_checked(&self.remote_relay().revocation, RelayLock::Revocation)?; - let peers = lock_checked(&self.remote_relay().peers, RelayLock::Peers)?; + let peers = self.remote_relay().peers_checked()?; let mut closing: Vec = peers .values() .filter(|peer| revocation.policy(&peer.install) == StreamPolicy::Close) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs index 0586005c8e68..d7df660c3246 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs @@ -1004,28 +1004,39 @@ fn a_poisoned_pairing_lock_denies_the_owner_scope() { } /// A poisoned peers lock cannot say which clients have a peer record, so -/// no client gets a principal from it and none is trusted as local. +/// no client gets a principal from it and none is trusted as local. The +/// same on the input lane (R-DAEMON-STATE stage S1), whose readers load the +/// published records instead of the lock. #[test] fn a_poisoned_peers_lock_gives_no_principal_and_no_local_trust() { - let fixture = fixture(); - let client = remote(&fixture, "inst_1", OWNER); - poison(&fixture.mux.remote_relay().peers); - assert_eq!(fixture.mux.principal(client), None); - assert!(fixture.mux.is_remote_client(fixture.local)); - assert_eq!(fixture.mux.principal(fixture.local), None); - let late = fixture.mux.control_clients.register(ClientTransport::Remote, writer().0); - assert_eq!( - fixture.mux.bind_remote_peer(late, &peer("inst_1", OWNER)), - Err(BindRefused::State(POISONED_PEERS)) - ); + for lane in [false, true] { + let fixture = fixture(); + fixture.mux.set_input_lane_for_test(lane); + let client = remote(&fixture, "inst_1", OWNER); + fixture.mux.remote_relay().peers.poison_for_test(); + assert_eq!(fixture.mux.principal(client), None, "lane {lane}"); + assert!(fixture.mux.is_remote_client(fixture.local), "lane {lane}"); + assert_eq!(fixture.mux.principal(fixture.local), None, "lane {lane}"); + let late = fixture.mux.control_clients.register(ClientTransport::Remote, writer().0); + assert_eq!( + fixture.mux.bind_remote_peer(late, &peer("inst_1", OWNER)), + Err(BindRefused::State(POISONED_PEERS)), + "lane {lane}" + ); + } } +/// The same for the client registry, also on the input lane, whose readers +/// load the registry's published facts instead of its lock. #[test] fn a_poisoned_registry_lock_trusts_no_client_as_local() { - let fixture = fixture(); - fixture.mux.control_clients.state.poison_for_test(); - assert!(fixture.mux.control_clients.transport_of(fixture.local).is_none()); - assert!(fixture.mux.is_remote_client(fixture.local)); + for lane in [false, true] { + let fixture = fixture(); + fixture.mux.set_input_lane_for_test(lane); + fixture.mux.control_clients.state.poison_for_test(); + assert!(fixture.mux.control_clients.transport_of(fixture.local).is_none(), "lane {lane}"); + assert!(fixture.mux.is_remote_client(fixture.local), "lane {lane}"); + } } /// A revoke that meets a poisoned lock cannot tell the install's streams @@ -1035,7 +1046,7 @@ fn a_revoke_on_a_poisoned_lock_closes_every_remote_stream() { let fixture = fixture(); let one = remote(&fixture, "inst_1", OWNER); let other = remote(&fixture, "inst_2", OWNER); - poison(&fixture.mux.remote_relay().peers); + fixture.mux.remote_relay().peers.poison_for_test(); assert_eq!(fixture.mux.revoke_remote_install("inst_1"), Err(POISONED_PEERS)); assert!(!fixture.mux.control_clients.is_remote(one)); assert!(!fixture.mux.control_clients.is_remote(other)); diff --git a/cmux-tui/crates/cmux-tui-remote-access/Cargo.toml b/cmux-tui/crates/cmux-tui-remote-access/Cargo.toml index 44befd04a5a1..e9048cb7d951 100644 --- a/cmux-tui/crates/cmux-tui-remote-access/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-remote-access/Cargo.toml @@ -18,4 +18,5 @@ test-support = [] anyhow.workspace = true base64.workspace = true cmux-link.workspace = true +cmux-tui-util.workspace = true getrandom.workspace = true diff --git a/cmux-tui/crates/cmux-tui-remote-access/src/remote_relay_state.rs b/cmux-tui/crates/cmux-tui-remote-access/src/remote_relay_state.rs index 9d9cd05e1215..1eefdc6d3981 100644 --- a/cmux-tui/crates/cmux-tui-remote-access/src/remote_relay_state.rs +++ b/cmux-tui/crates/cmux-tui-remote-access/src/remote_relay_state.rs @@ -12,6 +12,10 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::{Duration, Instant}; pub use cmux_link::stamp::LinkPeer; +use cmux_tui_util::published::{PublishGuard, Published}; + +/// The verified link peer of each remote connection, by client id. +pub type PeerRecords = BTreeMap; /// Last good check older than this: new streams of the install are refused. pub const REFUSE_NEW_STREAMS_AFTER: Duration = Duration::from_secs(24 * 60 * 60); @@ -225,8 +229,10 @@ pub fn lock_checked( /// The remote-relay state on the conversation host. #[derive(Default)] pub struct RemoteRelayState { - /// The verified link peer of each remote connection (memory only). - pub peers: Mutex>, + /// The verified link peer of each remote connection (memory only). Every line of every connection asks for its + /// record (the remote-client check, the request's actor), so with publishing on (R-DAEMON-STATE stage S1) a + /// reader loads the published copy and never waits for a bind, a disconnect or a revocation sweep. + pub peers: Published, /// Connections whose peer stamp was produced after the Cloud control /// plane accepted a link token. A caller-supplied capability is not /// authority; Cloud mobile uses this authenticated admission bit. @@ -236,10 +242,33 @@ pub struct RemoteRelayState { } impl RemoteRelayState { + /// The state of a daemon whose stage S1 switch (R-DAEMON-STATE) is `publish_peers`: on, the peer records are + /// published and [`Self::peer_checked`] never takes their lock. + pub fn new(publish_peers: bool) -> Self { + Self { + peers: Published::new(PeerRecords::new(), PeerRecords::clone, publish_peers), + ..Self::default() + } + } + + /// The peer records, locked for a change or for a read of more than one record; an error when a panic poisoned + /// them (callers fail closed). + pub fn peers_checked( + &self, + ) -> Result, RelayStateError> { + self.peers.lock().map_err(|_| RelayStateError::Poisoned(RelayLock::Peers)) + } + /// The peer record of `client`; an error when the peers lock is - /// poisoned (callers fail closed). + /// poisoned (callers fail closed). With publishing on, the published + /// records: the newest finished bind or removal, no wait. pub fn peer_checked(&self, client: u64) -> Result, RelayStateError> { - Ok(lock_checked(&self.peers, RelayLock::Peers)?.get(&client).cloned()) + if self.peers.publishing() { + let peers = + self.peers.load().map_err(|_| RelayStateError::Poisoned(RelayLock::Peers))?; + return Ok(peers.get(&client).cloned()); + } + Ok(self.peers_checked()?.get(&client).cloned()) } /// The peer record of `client` (tests). A poisoned peers lock gives @@ -258,7 +287,7 @@ impl RemoteRelayState { /// The remote connections of `install`. pub fn clients_of(&self, install: &str) -> Result, RelayStateError> { - let peers = lock_checked(&self.peers, RelayLock::Peers)?; + let peers = self.peers_checked()?; Ok(peers.iter().filter(|(_, peer)| peer.install == install).map(|(c, _)| *c).collect()) } diff --git a/cmux-tui/crates/cmux-tui-util/Cargo.toml b/cmux-tui/crates/cmux-tui-util/Cargo.toml index 368305effb83..2be488cfd6ea 100644 --- a/cmux-tui/crates/cmux-tui-util/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-util/Cargo.toml @@ -5,16 +5,17 @@ edition.workspace = true rust-version.workspace = true license.workspace = true publish.workspace = true -description = "Small cmux-tui daemon primitives without daemon state: accept backoff, stream interrupts, debug spans, short ids, machine name, respawn text, settings-file reads" +description = "Small cmux-tui daemon primitives without daemon state: accept backoff, stream interrupts, debug spans, short ids, machine name, respawn text, settings-file reads, published values" [lints] workspace = true [features] -# Test seams (the scoped new-workspace placement), for dev-dependencies only. +# Test seams (the scoped new-workspace placement, published-value holds and poison), for dev-dependencies only. test-support = [] [dependencies] +arc-swap.workspace = true cmux-tui-platform.workspace = true getrandom.workspace = true libc.workspace = true diff --git a/cmux-tui/crates/cmux-tui-util/src/lib.rs b/cmux-tui/crates/cmux-tui-util/src/lib.rs index 20a3417f5403..ad6c2a11ee72 100644 --- a/cmux-tui/crates/cmux-tui-util/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-util/src/lib.rs @@ -1,9 +1,10 @@ //! Small primitives of the cmux-tui daemon that hold no daemon state: accept //! retry backoff ([`backoff`]), stream interruption ([`stream_interrupt`]), //! terminal-create debug spans ([`debug_spans`]), short ids, this machine's -//! name, the respawn marker text, and the few settings-file keys the daemon -//! reads ([`user_settings`]). cmux-tui-core re-exports each module at its old -//! path (`crate::backoff`, ...). +//! name, the respawn marker text, the few settings-file keys the daemon +//! reads ([`user_settings`]), and a mutex-guarded value whose readers load a +//! published copy without the lock ([`published`]). cmux-tui-core re-exports +//! each module at its old path (`crate::backoff`, ...). /// The OS paths `user_settings` reads (`crate::platform::home_dir`). use cmux_tui_platform::platform; @@ -11,6 +12,7 @@ use cmux_tui_platform::platform; pub mod backoff; pub mod debug_spans; pub mod machine_name; +pub mod published; pub mod short_id; pub mod stream_interrupt; pub mod terminal_respawn_text; diff --git a/cmux-tui/crates/cmux-tui-core/src/published.rs b/cmux-tui/crates/cmux-tui-util/src/published.rs similarity index 60% rename from cmux-tui/crates/cmux-tui-core/src/published.rs rename to cmux-tui/crates/cmux-tui-util/src/published.rs index 66cc06b613f9..dda851066718 100644 --- a/cmux-tui/crates/cmux-tui-core/src/published.rs +++ b/cmux-tui/crates/cmux-tui-util/src/published.rs @@ -10,31 +10,42 @@ //! The projection is the only copy a reader sees, and only the guard writes it: no code path can //! change the value without publishing the change, because every change goes through the guard. //! +//! A change that a panic cuts short is never published: the guard that held it drops during the unwind, and the +//! mutex is then marked poisoned, so readers fail closed with the last finished projection. +//! //! Publishing is on or off for the value's life (a stage switch decides it when the value is made; //! tests can turn it on). Off, a guard builds no projection, so a value nobody reads lock-free costs -//! what a plain mutex costs. +//! what a plain mutex costs; a reader asks [`Published::publishing`] which read is current. use std::ops::{Deref, DerefMut}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, LockResult, Mutex, MutexGuard, PoisonError}; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] use std::sync::{TryLockError, TryLockResult}; use arc_swap::ArcSwap; /// A mutex-guarded value with a lock-free published projection (see the module docs). -pub(crate) struct Published { +pub struct Published { value: Mutex, projection: ArcSwap, project: fn(&T) -> S, - /// Whether a changing guard publishes; read and written only under `value`'s lock. + /// Whether a changing guard publishes. Fixed when the value is made; a test may set it, under `value`'s lock and + /// after the current value is published (Release), so a reader that sees it on (Acquire) sees a current projection. publishing: AtomicBool, } +/// A value whose projection is a copy of the whole value (small maps the key path reads), not publishing. +impl Default for Published { + fn default() -> Self { + Self::new(T::default(), T::clone, false) + } +} + impl Published { /// `project` makes the immutable projection that readers load. It runs under the mutex after /// each change while `publishing` is on, so it must read only and stay small. - pub(crate) fn new(value: T, project: fn(&T) -> S, publishing: bool) -> Self { + pub fn new(value: T, project: fn(&T) -> S, publishing: bool) -> Self { let projection = ArcSwap::from_pointee(project(&value)); Self { value: Mutex::new(value), @@ -45,61 +56,87 @@ impl Published { } /// Turns publishing on or off; on, the current value is published first, under the lock (tests). - #[cfg(test)] - pub(crate) fn set_publishing_for_test(&self, on: bool) { + #[cfg(any(test, feature = "test-support"))] + pub fn set_publishing_for_test(&self, on: bool) { let value = self.value.lock().unwrap_or_else(PoisonError::into_inner); if on { self.projection.store(Arc::new((self.project)(&*value))); } - self.publishing.store(on, Ordering::Relaxed); + self.publishing.store(on, Ordering::Release); + } + + /// Whether changes are published: on, [`Self::load`] gives the newest finished change; off, the projection is + /// stale and a reader takes [`Self::lock`] instead. + pub fn publishing(&self) -> bool { + self.publishing.load(Ordering::Acquire) + } + + /// Whether a writer panicked while it held the value. + pub fn is_poisoned(&self) -> bool { + self.value.is_poisoned() } /// The projection of the newest finished change: one atomic load, never a wait. When a writer /// panicked while it held the value (the mutex is poisoned), the projection comes in the error, /// as `std::sync::Mutex::lock` gives its guard, so a caller can fail closed. - pub(crate) fn load(&self) -> LockResult>> { + pub fn load(&self) -> LockResult>> { let projection = self.projection.load(); if self.value.is_poisoned() { Err(PoisonError::new(projection)) } else { Ok(projection) } } /// The value itself, for a change or for a read that needs more than the projection. A /// poisoned mutex gives its guard in the error, as `std::sync::Mutex::lock` does. - pub(crate) fn lock(&self) -> LockResult> { + pub fn lock(&self) -> LockResult> { match self.value.lock() { - Ok(guard) => Ok(PublishGuard { owner: self, guard, changed: false }), - Err(poison) => Err(PoisonError::new(PublishGuard { - owner: self, - guard: poison.into_inner(), - changed: false, - })), + Ok(guard) => Ok(self.guard(guard)), + Err(poison) => Err(PoisonError::new(self.guard(poison.into_inner()))), + } + } + + fn guard<'a>(&'a self, guard: MutexGuard<'a, T>) -> PublishGuard<'a, T, S> { + PublishGuard { + owner: self, + guard, + changed: false, + panicking_at_lock: std::thread::panicking(), + retired: None, } } } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] impl Published { /// The value if no thread holds it now (tests that prove a hold). - pub(crate) fn try_lock(&self) -> TryLockResult> { + pub fn try_lock(&self) -> TryLockResult> { match self.value.try_lock() { - Ok(guard) => Ok(PublishGuard { owner: self, guard, changed: false }), + Ok(guard) => Ok(self.guard(guard)), Err(TryLockError::WouldBlock) => Err(TryLockError::WouldBlock), Err(TryLockError::Poisoned(poison)) => { - Err(TryLockError::Poisoned(PoisonError::new(PublishGuard { - owner: self, - guard: poison.into_inner(), - changed: false, - }))) + Err(TryLockError::Poisoned(PoisonError::new(self.guard(poison.into_inner())))) } } } + + /// Poisons the value: a panic while it is held (fail-closed tests). + pub fn poison_for_test(&self) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _held = self.value.lock(); + std::panic::panic_any("poisons the value for a fail-closed test"); + })); + } } /// A held [`Published`] value. A mutable borrow marks it changed; it then publishes the new /// projection when it drops, before the mutex is released (fields drop after `Drop::drop`). -pub(crate) struct PublishGuard<'a, T, S> { +pub struct PublishGuard<'a, T, S> { owner: &'a Published, guard: MutexGuard<'a, T>, changed: bool, + /// Whether this thread was already unwinding when it took the value, as `std::sync::MutexGuard` records it. + panicking_at_lock: bool, + /// The projection this guard replaced. It drops after `guard` (fields drop in order), so what it may hold last + /// is freed after the mutex is released, never under it. + retired: Option>, } impl Deref for PublishGuard<'_, T, S> { @@ -119,9 +156,14 @@ impl DerefMut for PublishGuard<'_, T, S> { impl Drop for PublishGuard<'_, T, S> { fn drop(&mut self) { + // A panic that began while this guard held the value cut its change short: the `MutexGuard` field poisons + // the mutex right after this (the same test std makes), and the half-made value is never published, so no + // reader sees it, not even before the poisoning. + let cut_short = std::thread::panicking() && !self.panicking_at_lock; // The flag is read under the lock this guard holds, where it is also written. - if self.changed && self.owner.publishing.load(Ordering::Relaxed) { - self.owner.projection.store(Arc::new((self.owner.project)(&*self.guard))); + if self.changed && !cut_short && self.owner.publishing.load(Ordering::Relaxed) { + let projection = Arc::new((self.owner.project)(&*self.guard)); + self.retired = Some(self.owner.projection.swap(projection)); } } } @@ -213,21 +255,44 @@ mod tests { let published = Published::new(Counter::default(), project, true); assert!(published.load().is_ok()); published.poison_for_test(); + assert!(published.is_poisoned()); assert!(published.load().is_err(), "a reader can fail closed on a poisoned value"); } + /// A guard taken while the thread already unwinds (a cleanup in a `Drop`) finishes its change: it is published, + /// as std leaves such a mutex unpoisoned. + #[test] + fn a_change_made_during_an_unwind_is_published() { + struct Cleanup<'a>(&'a Published); + impl Drop for Cleanup<'_> { + fn drop(&mut self) { + self.0.lock().unwrap_or_else(|poison| poison.into_inner()).value = 5; + } + } + let published = Published::new(Counter::default(), project, true); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _cleanup = Cleanup(&published); + panic!("an unrelated panic runs the cleanup"); + })); + assert!(!published.is_poisoned()); + assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 5); + } + + /// Off, nothing is published and a reader is told so. + #[test] + fn a_value_that_does_not_publish_says_so() { + let published = Published::new(Counter::default(), project, false); + assert!(!published.publishing()); + published.lock().unwrap_or_else(|poison| poison.into_inner()).value = 3; + assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 0); + published.set_publishing_for_test(true); + assert!(published.publishing()); + assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 3); + } + impl Published { fn load_full_for_test(&self) -> std::sync::Arc { self.projection.load_full() } - - /// Poisons the value: a panic while it is held (fail-closed tests). - pub(crate) fn poison_for_test(&self) { - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _held = self.value.lock(); - panic!("poisons the value for a fail-closed test"); - })); - assert!(self.value.is_poisoned()); - } } } From eecb0d813a2029856a9bf97440150fd25e8ec213 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:03:45 +0000 Subject: [PATCH 11/42] test(cmux-tui-core): a key waits for no creating-input lock, and for no session lock behind one (R-DAEMON-STATE S1 step 4 red) Every key to a live terminal first asks whether creating input of its terminal must go first (wait_for_creating_input_flush, DESIGN.md Appendix A.2 step 16). It takes the daemon-wide creating-input lock even when no terminal is being created, and that lock's holders (begin_creating_input, knows_terminal_input_target, finish_creating_input) look up the live terminal under it, which takes the session lock. The S1 census measured a full 300 ms wait on the creating-input lock (hosted run 41648095321: 300.3 ms). Two holders, each with an old-path control: the creating-input lock itself, and the session lock held while a reader waits for it under the creating-input lock (with another terminal being created, so the key cannot skip the lock). Red on this commit: the stage does not change these paths yet. --- .../cmux-tui-core/src/mux/creating_input.rs | 16 ++++++ .../src/server/tests/rds_input_lane.rs | 51 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs index 3ae76d43f380..72438ecd2241 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs @@ -312,6 +312,22 @@ impl Mux { } } + /// Runs `hold` while this thread holds the creating-input lock (tests: a person's key must not wait for it). + #[cfg(test)] + pub(crate) fn hold_creating_inputs_for_test(&self, hold: &dyn Fn()) { + let _held = self.creating_inputs.terminals.lock().unwrap_or_else(PoisonError::into_inner); + hold(); + } + + /// Whether a thread holds the creating-input lock now (tests). + #[cfg(test)] + pub(crate) fn creating_inputs_held_for_test(&self) -> bool { + matches!( + self.creating_inputs.terminals.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + ) + } + /// The live terminal surface whose host id is `terminal_id`. #[track_caller] fn live_surface_for_terminal(&self, terminal_id: &str) -> Option> { diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index 3b3a0ea10d70..d2d66e243f85 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -164,6 +164,42 @@ fn hold_proxy_authors(mux: &Arc, hold: &dyn Fn()) { mux.control_clients.agent_sessions.hold_proxy_authors_for_test(hold); } +fn hold_creating_inputs(mux: &Arc, hold: &dyn Fn()) { + mux.hold_creating_inputs_for_test(hold); +} + +/// A terminal another creation names, still being created (`terminal-input-while-creating-v1`). +const CREATING_TERMINAL: &str = "000000000000000000000000c4ea7e01"; +/// A terminal nobody creates: a reader asks about it under the creating-input lock. +const OTHER_TERMINAL: &str = "000000000000000000000000c4ea7e02"; + +/// While another terminal is being created, one thread holds the session lock and another asks about a terminal +/// under the creating-input lock. The old path looked up the live terminal under that lock, so the reader waited +/// for the session lock while it held the creating-input lock, and a key behind it waited too. +fn hold_session_behind_a_creating_input_reader(mux: &Arc, hold: &dyn Fn()) { + let generation = + mux.begin_creating_input(CREATING_TERMINAL).expect("no live terminal has the id"); + let mut reader = None; + mux.with_state(|_| { + let asking = mux.clone(); + let asked = std::thread::spawn(move || asking.knows_terminal_input_target(OTHER_TERMINAL)); + // Until the reader holds the creating-input lock (the old path waits under it) or has its answer. + let deadline = Instant::now() + Duration::from_secs(1); + while !asked.is_finished() + && !mux.creating_inputs_held_for_test() + && Instant::now() < deadline + { + std::thread::yield_now(); + } + reader = Some(asked); + hold(); + }); + if let Some(reader) = reader { + let _ = reader.join(); + } + mux.finish_creating_input(CREATING_TERMINAL, generation, Some("test")); +} + /// How long two keys took to reach their PTY while another thread held one lock (the later key's arrival from the /// first key's submit; `Duration::MAX` when a key never arrived), on a connection that `connection` is. fn key_wait_while_held(connection: &TypingConnection, index: usize, holder: Holder) -> Duration { @@ -276,6 +312,21 @@ fn a_key_waits_for_neither_the_link_peers_nor_the_proxy_authors_lock() { ]); } +/// Stage S1 step 4: a key asks whether creating input of its terminal must go first. With no terminal being created +/// it reads an atomic count and takes no lock; with one, the creating-input lock is a leaf: no holder waits under it +/// for the session lock (its live-terminal lookups read the published routes). +#[test] +fn a_key_waits_for_no_creating_input_lock() { + let creating = "creating input (CreatingInputs.terminals)"; + let behind = "session (Mux.state) behind a creating-input reader"; + assert_old_path_waits_for(creating, hold_creating_inputs); + assert_old_path_waits_for(behind, hold_session_behind_a_creating_input_reader); + assert_keys_wait_for_none(&[ + (creating, hold_creating_inputs as Holder), + (behind, hold_session_behind_a_creating_input_reader as Holder), + ]); +} + /// #31940 and TYPING FIX 2 took these two off the key; this keeps them off on the stage's new path. #[test] fn a_key_waits_for_neither_the_session_nor_the_workspace_registry_lock() { From 8a502885dc1142968abbdef7e0010fcf70225064 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:06:16 +0000 Subject: [PATCH 12/42] fix(cmux-tui-core): a key skips the creating-input lock while nothing is creating; the lock becomes a leaf (R-DAEMON-STATE S1 step 4) On the input lane (CMUX_TUI_RDS_INPUT, per mux, default off): - CreatingInputs keeps the number of creating terminals in an atomic, stored under its lock after every insert and removal (Release). A key reads it (Acquire) and returns at once when it is zero: no queue can be ahead of it, because a terminal's queue is added before the terminal goes live and removed only after every queued byte was written to it. - The creating-input lock's holders look up the live terminal in the published routes instead of under the session lock, so no holder waits for the session lock while it holds the creating-input lock, and a key that takes it never waits behind one. The routes carry the host-id catalog index for this: terminal_catalog_by_host becomes a copy-on-write route table in the RouteSnapshot, and one lookup (route_table::live_terminal) serves the session state and the snapshot. A commit publishes its routes before it releases the session lock, and finish runs after the commit, so begin, knows and finish give the answers they gave under the session lock. Off, every path takes the locks it took on the base; the by-host index is copy-on-write on both arms, as the other three route tables are since step 1. --- cmux-tui/crates/cmux-tui-core/src/model.rs | 5 +- .../cmux-tui-core/src/mux/creating_input.rs | 75 ++++++++++++++----- .../src/mux/public_projections/tests.rs | 2 +- .../crates/cmux-tui-core/src/mux/restore.rs | 2 +- .../src/mux/terminal_catalog_index.rs | 11 ++- .../crates/cmux-tui-core/src/route_table.rs | 46 ++++++++++++ 6 files changed, 111 insertions(+), 30 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/model.rs b/cmux-tui/crates/cmux-tui-core/src/model.rs index a565ca12193b..571c4e1b3712 100644 --- a/cmux-tui/crates/cmux-tui-core/src/model.rs +++ b/cmux-tui/crates/cmux-tui-core/src/model.rs @@ -641,8 +641,9 @@ pub struct State { /// Reverse lookup for catalog owners addressed by daemon-local runtime ID. pub(crate) terminal_catalog_by_runtime: crate::route_table::RouteTable, - /// Host terminal id -> catalog owner (mux/terminal_catalog_index.rs). - pub(crate) terminal_catalog_by_host: HashMap, + /// Host terminal id -> catalog owner (mux/terminal_catalog_index.rs). Copy-on-write and published with the + /// routes, so the live-terminal lookup reads it with no lock (R-DAEMON-STATE stage S1). + pub(crate) terminal_catalog_by_host: crate::route_table::RouteTable, pub(crate) split_screens: HashMap, pub(crate) resource_indexes: PublicSlotIndexes, } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs index 72438ecd2241..776106b8a8d9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs @@ -51,10 +51,38 @@ struct CreatingTerminal { #[derive(Default)] pub(crate) struct CreatingInputs { terminals: Mutex>, + /// How many terminals are creating: the map's length, stored under its lock after each change. A key reads it + /// with no lock and takes the lock only while a terminal is creating (R-DAEMON-STATE stage S1). + creating: AtomicUsize, ended: Condvar, next_generation: AtomicU64, } +impl CreatingInputs { + /// Adds the creating terminal `terminal_id`; the count follows, under the lock. + fn insert( + &self, + terminals: &mut HashMap, + terminal_id: &str, + creating: CreatingTerminal, + ) { + terminals.insert(terminal_id.to_string(), creating); + self.creating.store(terminals.len(), Ordering::Release); + } + + /// Removes the creating terminal `terminal_id` (its queue was written or given back); the count follows, under + /// the lock, after every byte of the queue went to the terminal. + fn remove( + &self, + terminals: &mut HashMap, + terminal_id: &str, + ) -> Option { + let removed = terminals.remove(terminal_id); + self.creating.store(terminals.len(), Ordering::Release); + removed + } +} + /// Where `write_terminal_input` put the bytes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum TerminalInputTarget { @@ -108,8 +136,9 @@ impl Mux { return None; } let generation = inputs.next_generation.fetch_add(1, Ordering::Relaxed); - terminals.insert( - terminal_id.to_string(), + inputs.insert( + &mut terminals, + terminal_id, CreatingTerminal { generation, queued: Vec::new(), @@ -175,6 +204,12 @@ impl Mux { /// when no terminal is creating. The caller holds no mux lock. pub(crate) fn wait_for_creating_input_flush(&self, surface: &Surface) { let inputs = &self.creating_inputs; + // No terminal is creating, so no queue can be ahead of this input: the count says so with no lock + // (R-DAEMON-STATE stage S1). A queue of this surface's terminal was added before the terminal went live, + // and the count drops to zero only after the last queue was written (Release, read here with Acquire). + if self.control_clients.input_lane.on() && inputs.creating.load(Ordering::Acquire) == 0 { + return; + } let mut terminals = inputs.terminals.lock().unwrap_or_else(PoisonError::into_inner); if terminals.is_empty() { return; @@ -219,7 +254,7 @@ impl Mux { let live = if failure.is_none() { self.live_surface_for_terminal(terminal_id) } else { None }; let Some(surface) = live else { - let queued = terminals.remove(terminal_id).map(|creating| creating.queued); + let queued = inputs.remove(&mut terminals, terminal_id).map(|creating| creating.queued); inputs.ended.notify_all(); drop(terminals); return give_back(terminal_id, queued.unwrap_or_default(), failure.unwrap_or("failed")); @@ -236,7 +271,7 @@ impl Mux { // A flushing entry stays until this loop removes it. let Some(creating) = terminals.get_mut(terminal_id) else { break }; if creating.queued.is_empty() { - terminals.remove(terminal_id); + inputs.remove(&mut terminals, terminal_id); inputs.ended.notify_all(); break; } @@ -258,7 +293,7 @@ impl Mux { let later = { let mut terminals = inputs.terminals.lock().unwrap_or_else(PoisonError::into_inner); - let creating = terminals.remove(terminal_id); + let creating = inputs.remove(&mut terminals, terminal_id); inputs.ended.notify_all(); creating.map(|creating| creating.queued).unwrap_or_default() }; @@ -328,24 +363,24 @@ impl Mux { ) } - /// The live terminal surface whose host id is `terminal_id`. + /// The live terminal surface whose host id is `terminal_id`. The creating-input lock's holders call it under + /// that lock; on the input lane (R-DAEMON-STATE stage S1) it reads the routes of the last finished change, so no + /// holder waits for the session lock under it and a key that takes it never waits behind one. A commit publishes + /// its routes before it releases the session lock, and a queue is removed only after its terminal is live, so the + /// creation checks give the answers they gave under the session lock. #[track_caller] fn live_surface_for_terminal(&self, terminal_id: &str) -> Option> { - let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(surface) = state.indexed_catalog_terminal(terminal_id) - && state.surfaces.contains_key(&surface.id) - { - return Some(surface.clone()); + if self.control_clients.input_lane.on() { + return self.state.routes().live_terminal(terminal_id); } - state - .surfaces - .values() - .find(|surface| { - surface - .terminal_host_identity() - .is_some_and(|identity| identity.terminal_id == terminal_id) - }) - .cloned() + let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + crate::route_table::live_terminal( + &state.surfaces, + &state.terminal_catalog, + &state.terminal_catalog_by_host, + terminal_id, + ) + .cloned() } } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections/tests.rs b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections/tests.rs index 1f4a76000e2a..b0de5070d5b4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections/tests.rs @@ -24,7 +24,7 @@ fn empty_state() -> State { surfaces: Default::default(), terminal_catalog: Default::default(), terminal_catalog_by_runtime: Default::default(), - terminal_catalog_by_host: HashMap::new(), + terminal_catalog_by_host: Default::default(), split_screens: HashMap::new(), resource_indexes: PublicSlotIndexes::default(), } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/restore.rs b/cmux-tui/crates/cmux-tui-core/src/mux/restore.rs index 3c3e57cabc7c..696e95d1080a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/restore.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/restore.rs @@ -330,7 +330,7 @@ pub(super) fn restore_resource_state( surfaces: Default::default(), terminal_catalog: Default::default(), terminal_catalog_by_runtime: Default::default(), - terminal_catalog_by_host: HashMap::new(), + terminal_catalog_by_host: Default::default(), split_screens: HashMap::new(), resource_indexes: indexes, }, diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_catalog_index.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_catalog_index.rs index b90ab11918f7..0fa60f20aed7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_catalog_index.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_catalog_index.rs @@ -28,12 +28,11 @@ impl State { /// The catalog owner indexed under host id `terminal_id`, if the index /// names one that still carries that host identity. pub(super) fn indexed_catalog_terminal(&self, terminal_id: &str) -> Option<&Arc> { - let public_id = self.terminal_catalog_by_host.get(terminal_id)?; - self.terminal_catalog.get(public_id).filter(|surface| { - surface - .terminal_host_identity() - .is_some_and(|identity| identity.terminal_id == terminal_id) - }) + crate::route_table::indexed_owner( + &self.terminal_catalog, + &self.terminal_catalog_by_host, + terminal_id, + ) } /// Insert a catalog owner and index its host id. A different runtime diff --git a/cmux-tui/crates/cmux-tui-core/src/route_table.rs b/cmux-tui/crates/cmux-tui-core/src/route_table.rs index b5039b91f657..ae77bf7e00c1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/route_table.rs +++ b/cmux-tui/crates/cmux-tui-core/src/route_table.rs @@ -85,6 +85,13 @@ impl RouteKey for SurfaceId { } } +/// `State::terminal_catalog_by_host`: the host id names a catalog owner, never a placement or runtime id itself. +impl RouteKey for String { + fn routed_id(&self, _: &TerminalPublicId) -> Option { + None + } +} + /// `State::terminal_catalog`: the runtime id the owner is found under (`terminal_catalog_by_runtime`). impl RouteKey> for TerminalPublicId { fn routed_id(&self, value: &Arc) -> Option { @@ -164,6 +171,7 @@ pub(crate) struct RouteSnapshot { surfaces: Arc>>, catalog: Arc>>, by_runtime: Arc>, + by_host: Arc>, } impl RouteSnapshot { @@ -173,6 +181,7 @@ impl RouteSnapshot { surfaces: state.surfaces.shared(), catalog: state.terminal_catalog.shared(), by_runtime: state.terminal_catalog_by_runtime.shared(), + by_host: state.terminal_catalog_by_host.shared(), } } @@ -181,6 +190,7 @@ impl RouteSnapshot { state.surfaces.shares(&self.surfaces) && state.terminal_catalog.shares(&self.catalog) && state.terminal_catalog_by_runtime.shares(&self.by_runtime) + && state.terminal_catalog_by_host.shares(&self.by_host) } /// The surface `id` routes to: its placement, else the catalog owner of the runtime `id` names (the lookup of @@ -191,6 +201,42 @@ impl RouteSnapshot { .or_else(|| self.by_runtime.get(&id).and_then(|terminal| self.catalog.get(terminal))) .cloned() } + + /// The live terminal whose host id is `terminal_id` (the lookup of `Mux::live_surface_for_terminal`). + pub(crate) fn live_terminal(&self, terminal_id: &str) -> Option> { + live_terminal(&self.surfaces, &self.catalog, &self.by_host, terminal_id).cloned() + } +} + +/// The catalog owner the host index names for host id `terminal_id`, if it still carries that identity: the index +/// is a verified cache (mux/terminal_catalog_index.rs). +pub(crate) fn indexed_owner<'a>( + catalog: &'a HashMap>, + by_host: &HashMap, + terminal_id: &str, +) -> Option<&'a Arc> { + let public_id = by_host.get(terminal_id)?; + catalog.get(public_id).filter(|surface| { + surface.terminal_host_identity().is_some_and(|identity| identity.terminal_id == terminal_id) + }) +} + +/// The live terminal whose host id is `terminal_id`: the indexed catalog owner while it is placed, else the placement +/// whose terminal has that identity. The session state and a published snapshot answer with these same tables. +pub(crate) fn live_terminal<'a>( + surfaces: &'a HashMap>, + catalog: &'a HashMap>, + by_host: &HashMap, + terminal_id: &str, +) -> Option<&'a Arc> { + if let Some(owner) = indexed_owner(catalog, by_host, terminal_id) + && surfaces.contains_key(&owner.id) + { + return Some(owner); + } + surfaces.values().find(|surface| { + surface.terminal_host_identity().is_some_and(|identity| identity.terminal_id == terminal_id) + }) } impl Deref for RouteTable { From 831686b8c7988d3ded548e1a8be58450862d017b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:30:30 +0000 Subject: [PATCH 13/42] test(cmux-tui-core): the surface budget's limits on both arms; a lock control fails on a lost key (#34146 review follow-ups) The #34146 review (comment 6613345630) noted: - Every budget test used the mutex arm, so an atomic arm that admitted everything would have passed. The two mux-wide limit tests now run on both arms, and a new test checks the same limits on both: the 17th worker, the whole byte budget, one byte over, zero bytes, a sum that overflows, and released reservations coming back. - The old-path control passed when a key never arrived (Duration::MAX >= BUDGET); it now fails on a lost key. - ServerSurfaceOperationState and the admission's mutex are private (nothing outside the file reads them), and the doc says which requests reserve (pings, identify, v2 resource lines and reader-thread frame handlers do not). --- .../crates/cmux-tui-core/src/server/tests.rs | 10 +++- .../server/tests/outbound_and_scheduler.rs | 57 ++++++++++++++++--- .../src/server/tests/rds_input_lane.rs | 4 ++ .../src/server/worker_admission.rs | 10 ++-- 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests.rs index 7eb5706607b4..2526b6048ea6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests.rs @@ -444,10 +444,16 @@ fn test_stream_id(index: u64) -> StreamPublicId { .expect("test stream id uses the public wire format") } -fn active_clear_lanes_across_connections(request_count: usize, retained_bytes: usize) -> usize { +/// How many clear-history requests run at once across two connections that share one admission budget; `lock_free` +/// picks the budget's arm (R-DAEMON-STATE stage S1: atomics on, the mutex counts off). +fn active_clear_lanes_across_connections( + request_count: usize, + retained_bytes: usize, + lock_free: bool, +) -> usize { let mux = test_mux(); let writer = test_writer(); - let admission = Arc::new(ServerSurfaceOperationAdmission::default()); + let admission = Arc::new(ServerSurfaceOperationAdmission::new(lock_free)); let schedulers = [ Arc::new(ConnectionSurfaceScheduler::new(admission.clone())), Arc::new(ConnectionSurfaceScheduler::new(admission)), diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs index b43e332056b5..2cf6e3cd86b5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/outbound_and_scheduler.rs @@ -1016,21 +1016,62 @@ fn scheduler_retains_connection_permit_until_dispatcher_exit() { assert_eq!(active.active(), 0); } +/// On both arms of the budget: the mutex counts, and the input lane's atomics (R-DAEMON-STATE stage S1). #[test] fn surface_worker_limit_is_mux_wide_across_connections() { - assert!( - active_clear_lanes_across_connections(17, 0) <= 16, - "per-connection limits allowed more than 16 mux-wide clear workers" - ); + for lock_free in [false, true] { + assert!( + active_clear_lanes_across_connections(17, 0, lock_free) <= 16, + "per-connection limits allowed more than 16 mux-wide clear workers (lock-free {lock_free})" + ); + } } #[test] fn active_surface_request_bytes_count_toward_mux_budget() { const FOUR_MIB: usize = 4 * 1024 * 1024; - assert!( - active_clear_lanes_across_connections(5, FOUR_MIB) <= 4, - "active first requests bypassed the 16 MiB mux-wide byte budget" - ); + for lock_free in [false, true] { + assert!( + active_clear_lanes_across_connections(5, FOUR_MIB, lock_free) <= 4, + "active first requests bypassed the 16 MiB mux-wide byte budget (lock-free {lock_free})" + ); + } +} + +/// The input lane's atomic budget admits what the mutex counts admit and refuses what they refuse: the 17th worker, +/// a byte reservation over 16 MiB or one whose sum would overflow, and gives a released reservation back. +#[test] +fn the_lock_free_budget_has_the_same_limits_as_the_mutex_counts() { + for lock_free in [false, true] { + let admission = Arc::new(ServerSurfaceOperationAdmission::new(lock_free)); + let workers = (0..SERVER_SURFACE_WORKER_CAPACITY) + .map(|_| admission.try_reserve_worker()) + .collect::>>() + .expect("the capacity's workers are admitted"); + assert!(admission.try_reserve_worker().is_none(), "lock-free {lock_free}: a 17th worker"); + drop(workers); + assert!( + admission.try_reserve_worker().is_some(), + "lock-free {lock_free}: released workers" + ); + + let all = admission.try_reserve_bytes(SERVER_SURFACE_RETAINED_BYTE_CAPACITY).ok(); + assert!(all.is_some(), "lock-free {lock_free}: the whole byte budget"); + assert!(admission.try_reserve_bytes(1).is_err(), "lock-free {lock_free}: a byte over"); + assert!(admission.try_reserve_bytes(0).is_ok(), "lock-free {lock_free}: zero bytes"); + drop(all); + let one = admission.try_reserve_bytes(1).ok(); + assert!(one.is_some(), "lock-free {lock_free}: one byte"); + assert!( + admission.try_reserve_bytes(usize::MAX).is_err(), + "lock-free {lock_free}: a sum that overflows" + ); + drop(one); + assert!( + admission.try_reserve_bytes(SERVER_SURFACE_RETAINED_BYTE_CAPACITY).is_ok(), + "lock-free {lock_free}: released bytes" + ); + } } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index d2d66e243f85..af41bd24a238 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -269,6 +269,10 @@ fn assert_old_path_waits_for(name: &str, holder: Holder) { let waited = key_wait_while_held(&connection, 0, holder); connection.close(); eprintln!("SHARED-LOCKS control (lane off) key wait while {name} is held {HOLD:?}: {waited:?}"); + assert!( + waited != Duration::MAX, + "control: with stage S1 off a key never reached its terminal while {name} was held" + ); assert!( waited >= BUDGET, "control: with stage S1 off the key took {waited:?} while {name} was held {HOLD:?}; the hold does not reach \ diff --git a/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs b/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs index f34fc056c30e..f3f82a602ac7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/worker_admission.rs @@ -110,13 +110,15 @@ impl ResourceWorkerAdmission { } #[derive(Default)] -pub(super) struct ServerSurfaceOperationState { - pub(super) workers: usize, +struct ServerSurfaceOperationState { + workers: usize, retained_bytes: usize, } /// Daemon-wide admission of surface operations: clear-history workers by count and queued requests by retained bytes. -/// Every request of every connection reserves here before it is queued and releases after it ran. +/// Every request a connection's surface scheduler queues reserves here before it is queued and releases after it ran +/// (the person's `send` and `send-key` lines among them; pings, identify, v2 resource lines and the frame handlers +/// that answer on the reader thread do not reserve). /// /// R-DAEMON-STATE stage S1 (`CMUX_TUI_RDS_INPUT`): with the lane on, the counters are atomics with compare-and-swap /// reservations, so no request, and no key of the person, waits for another connection's reservation. With it off, @@ -124,7 +126,7 @@ pub(super) struct ServerSurfaceOperationState { #[derive(Default)] pub(crate) struct ServerSurfaceOperationAdmission { /// The old arm's counters (stage S1 off). - pub(super) state: Mutex, + state: Mutex, /// The lane's counters (stage S1 on). workers: AtomicUsize, retained_bytes: AtomicUsize, From 10f16693a03b3a70a6b77610dd526d84cdcb6033 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:57:34 +0000 Subject: [PATCH 14/42] fix(cmux-tui-core): drop an unnecessary path qualification in a test (clippy unused_qualifications) The lint (linux) job's clippy -D warnings failed on mux/tests/signaled_mutex.rs:120 (TerminalPublicId is in scope); the #34174 review found it behind the base's acpmux red in the same job. --- cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs b/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs index 47daeb6cde6c..65514d9291c6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/tests/signaled_mutex.rs @@ -116,8 +116,7 @@ fn routes_of_a_change_cut_short_by_a_panic_are_never_published() { let state = test_mux().with_state(State::clone); let mutex = super::super::signaled_mutex::StateMutex::new(state, true); let before = arc_swap::Guard::into_inner(mutex.routes()); - let terminal = - crate::resource::TerminalPublicId::parse("term_00000000000000000000000000000077").unwrap(); + let terminal = TerminalPublicId::parse("term_00000000000000000000000000000077").unwrap(); let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let mut state = mutex.lock().unwrap_or_else(PoisonError::into_inner); state.terminal_catalog_by_runtime.insert(9_999_999, terminal); From 690b82fb3b8617e82f13014372d4a72801d76880 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:04:19 +0000 Subject: [PATCH 15/42] test(cmux-tui-core): a poisoned session state never routes by the published tables (R-DAEMON-STATE S1 red) The #34171 review noted: after a panic poisons Mux::state, the next guard that recovers the poison may publish the half-made route tables, and the lane's Mux::surface reads the published routes with no poison check, where the old path fails (it unwraps the lock). Red on this commit: with the lane on, a surface read after the poisoning returns from the published routes instead of failing. The relay poison tests now poison published values through a helper in the test file. --- .../src/server/remote_relay/tests.rs | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs index d7df660c3246..c542b2694570 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs @@ -956,6 +956,15 @@ fn a_remote_entry_connection_keeps_the_remote_path_after_disconnect() { // revocation step fails closed and returns a typed error. /// Poisons `mutex`: a panic while it is held. +/// Poisons a published value: a panic while its lock is held. +fn poison_published(value: &crate::published::Published) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _held = value.lock(); + panic!("poisons the value for a fail-closed test"); + })); + assert!(value.is_poisoned()); +} + fn poison(mutex: &Mutex) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let _held = mutex.lock().unwrap(); @@ -1013,7 +1022,7 @@ fn a_poisoned_peers_lock_gives_no_principal_and_no_local_trust() { let fixture = fixture(); fixture.mux.set_input_lane_for_test(lane); let client = remote(&fixture, "inst_1", OWNER); - fixture.mux.remote_relay().peers.poison_for_test(); + poison_published(&fixture.mux.remote_relay().peers); assert_eq!(fixture.mux.principal(client), None, "lane {lane}"); assert!(fixture.mux.is_remote_client(fixture.local), "lane {lane}"); assert_eq!(fixture.mux.principal(fixture.local), None, "lane {lane}"); @@ -1028,12 +1037,28 @@ fn a_poisoned_peers_lock_gives_no_principal_and_no_local_trust() { /// The same for the client registry, also on the input lane, whose readers /// load the registry's published facts instead of its lock. +/// A panic while the session state is held: with the lane on, the next surface read takes the old path and fails +/// as it does, never routing by tables that a guard which recovered the poison may have published. No terminal: +/// the mux has none, so nothing else locks the poisoned state. +#[test] +fn a_poisoned_session_state_never_routes_by_the_published_tables() { + let fixture = fixture(); + fixture.mux.set_input_lane_for_test(true); + assert!(fixture.mux.surface(424_242).is_none()); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + fixture.mux.with_state(|_| panic!("poisons the session state")); + })); + let read = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| fixture.mux.surface(424_242))); + assert!(read.is_err(), "the lane read the published routes of a poisoned session state"); +} + #[test] fn a_poisoned_registry_lock_trusts_no_client_as_local() { for lane in [false, true] { let fixture = fixture(); fixture.mux.set_input_lane_for_test(lane); - fixture.mux.control_clients.state.poison_for_test(); + poison_published(&fixture.mux.control_clients.state); assert!(fixture.mux.control_clients.transport_of(fixture.local).is_none(), "lane {lane}"); assert!(fixture.mux.is_remote_client(fixture.local), "lane {lane}"); } @@ -1046,7 +1071,7 @@ fn a_revoke_on_a_poisoned_lock_closes_every_remote_stream() { let fixture = fixture(); let one = remote(&fixture, "inst_1", OWNER); let other = remote(&fixture, "inst_2", OWNER); - fixture.mux.remote_relay().peers.poison_for_test(); + poison_published(&fixture.mux.remote_relay().peers); assert_eq!(fixture.mux.revoke_remote_install("inst_1"), Err(POISONED_PEERS)); assert!(!fixture.mux.control_clients.is_remote(one)); assert!(!fixture.mux.control_clients.is_remote(other)); @@ -1145,7 +1170,7 @@ fn a_remote_bind_on_a_poisoned_bindings_lock_refuses_without_a_panic() { fn a_poisoned_registry_trusts_no_unix_client_and_still_removes_a_record() { let fixture = fixture(); let one = remote(&fixture, "inst_1", OWNER); - fixture.mux.control_clients.state.poison_for_test(); + poison_published(&fixture.mux.control_clients.state); assert!(!fixture.mux.control_clients.is_unix(fixture.local)); assert_eq!(fixture.mux.principal(fixture.local), None); assert!(fixture.mux.control_clients.remove(one).is_some()); From bb0265f43d538e1170ca7a5b3f2af1b886ad9213 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:04:33 +0000 Subject: [PATCH 16/42] fix(cmux-tui-core): the lane's surface read takes the old path on a poisoned state; the poison helper is test code (#34171 review) Review comment 6614044759 (FIX at 8f7d40500e8): - published.rs: poison_for_test moved out of the test module with its panic! respelled as panic_any, which hides a hit from the crash ratchet's count. It is test code again: a private helper in the published tests and one in the relay tests, with plain panic!. - Mux::surface on the lane reads the published routes only while the session state is not poisoned; a poisoned state takes the old read and fails as it does (StateMutex::is_poisoned). A guard that recovers the poison may publish the half-made tables of the change that panicked. - Published::load debug-asserts that the value publishes: only a publishing value has a current projection. - The module doc says what load gives on a poisoned value; the caller decides whether to fail closed. The unused_qualifications error at mux/tests/signaled_mutex.rs:120 is fixed in 10f16693a03. --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 6 ++--- .../cmux-tui-core/src/mux/signaled_mutex.rs | 5 ++++ .../crates/cmux-tui-util/src/published.rs | 25 +++++++++++-------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d87524cf570d..a6d5d25327e0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -929,9 +929,9 @@ impl Mux { #[track_caller] pub fn surface(&self, id: SurfaceId) -> Option> { - if self.control_clients.input_lane.on() { - // The routing tables of the last finished change, with no lock (R-DAEMON-STATE stage S1): the same - // lookup, never a wait behind a list, a snapshot or a durable mutation that holds the session lock. + // The routes of the last finished change, with no lock (R-DAEMON-STATE stage S1); a poisoned state takes the + // old read and fails as it does (a guard that recovers the poison may publish half-made tables). + if self.control_clients.input_lane.on() && !self.state.is_poisoned() { return self.state.routes().surface(id); } let state = self.state.lock().unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs b/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs index cc4a500b2c82..9a418883839a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/signaled_mutex.rs @@ -355,6 +355,11 @@ impl StateMutex { self.publish_routes.store(on, std::sync::atomic::Ordering::Relaxed); } + /// Whether a writer panicked while it held the session state. + pub(crate) fn is_poisoned(&self) -> bool { + self.value.is_poisoned() + } + /// The routing tables as of the last finished change: one atomic load, never a wait (R-DAEMON-STATE stage S1). pub(crate) fn routes( &self, diff --git a/cmux-tui/crates/cmux-tui-util/src/published.rs b/cmux-tui/crates/cmux-tui-util/src/published.rs index dda851066718..3f05dc7fdd79 100644 --- a/cmux-tui/crates/cmux-tui-util/src/published.rs +++ b/cmux-tui/crates/cmux-tui-util/src/published.rs @@ -11,7 +11,8 @@ //! change the value without publishing the change, because every change goes through the guard. //! //! A change that a panic cuts short is never published: the guard that held it drops during the unwind, and the -//! mutex is then marked poisoned, so readers fail closed with the last finished projection. +//! mutex is then marked poisoned, so a reader's [`Published::load`] gives the last finished projection inside the +//! poison error, and the reader decides whether to fail closed or to recover it. //! //! Publishing is on or off for the value's life (a stage switch decides it when the value is made; //! tests can turn it on). Off, a guard builds no projection, so a value nobody reads lock-free costs @@ -78,8 +79,10 @@ impl Published { /// The projection of the newest finished change: one atomic load, never a wait. When a writer /// panicked while it held the value (the mutex is poisoned), the projection comes in the error, - /// as `std::sync::Mutex::lock` gives its guard, so a caller can fail closed. + /// as `std::sync::Mutex::lock` gives its guard, so a caller can fail closed. Only a value that + /// publishes has a current projection; a reader of one that does not takes [`Self::lock`]. pub fn load(&self) -> LockResult>> { + debug_assert!(self.publishing(), "a projection read of a value that does not publish"); let projection = self.projection.load(); if self.value.is_poisoned() { Err(PoisonError::new(projection)) } else { Ok(projection) } } @@ -116,14 +119,6 @@ impl Published { } } } - - /// Poisons the value: a panic while it is held (fail-closed tests). - pub fn poison_for_test(&self) { - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _held = self.value.lock(); - std::panic::panic_any("poisons the value for a fail-closed test"); - })); - } } /// A held [`Published`] value. A mutable borrow marks it changed; it then publishes the new @@ -284,7 +279,7 @@ mod tests { let published = Published::new(Counter::default(), project, false); assert!(!published.publishing()); published.lock().unwrap_or_else(|poison| poison.into_inner()).value = 3; - assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 0); + assert_eq!(**published.load_full_for_test(), 0, "a change was published while off"); published.set_publishing_for_test(true); assert!(published.publishing()); assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 3); @@ -294,5 +289,13 @@ mod tests { fn load_full_for_test(&self) -> std::sync::Arc { self.projection.load_full() } + + /// Poisons the value: a panic while it is held (fail-closed tests). + fn poison_for_test(&self) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _held = self.lock(); + panic!("poisons the value for a fail-closed test"); + })); + } } } From 3b2e29f1389abb1c8b36748fc1a4a5d839a87578 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:11:40 +0000 Subject: [PATCH 17/42] fix(cmux-tui-core): the creating-input live check reads the routes only on an unpoisoned state; its comment states the lane's linearization (#34174 review) Review comment 6613998735 (SHIP, notes): the begin_creating_input comment still said the check takes the state under the creating lock; on the lane it reads the published routes, and its answer is one the old path can give (a commit in progress has not made the id live, as when the old path takes the state before it). The lane's live lookup now takes the old read on a poisoned state, as Mux::surface does since bb0265f43d5. --- .../crates/cmux-tui-core/src/mux/creating_input.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs index 776106b8a8d9..cf18ac8ac31c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs @@ -127,8 +127,11 @@ impl Mux { /// it), or an earlier creation of the id is still in flight and owns it. pub(crate) fn begin_creating_input(&self, terminal_id: &str) -> Option { let inputs = &self.creating_inputs; - // The live check and the insert are one step under the creating lock (lock order: - // creating, then state), so a commit cannot make the id live in between. + // The live check and the insert are one step under the creating lock. Off the input lane the + // check takes the session state under it (lock order: creating, then state), so a commit + // cannot make the id live in between. On the lane it reads the published routes, the last + // finished change: a commit still in progress has not made the id live, as when the old path + // takes the state before that commit, so every answer is one the old path can give. let mut terminals = inputs.terminals.lock().unwrap_or_else(PoisonError::into_inner); if terminals.contains_key(terminal_id) || self.live_surface_for_terminal(terminal_id).is_some() @@ -370,7 +373,8 @@ impl Mux { /// creation checks give the answers they gave under the session lock. #[track_caller] fn live_surface_for_terminal(&self, terminal_id: &str) -> Option> { - if self.control_clients.input_lane.on() { + // A poisoned state takes the old read, as `Mux::surface` does. + if self.control_clients.input_lane.on() && !self.state.is_poisoned() { return self.state.routes().live_terminal(terminal_id); } let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); From dfe81bff4d8db8be4dcdaa7f5a76249750e21852 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:28:07 +0000 Subject: [PATCH 18/42] test(cmux-tui-core): a key waits for neither the sizing nor the activity lock; a key takes the grid back after another view's click (R-DAEMON-STATE S1 step 5 red) After its PTY write a key notes activity in the terminal's sizing (the newest view sets the grid under the default policy): it takes the daemon-wide client sizing lock twice (the note and the size-state publication), then the VM activity stream's lock. Every attach, detach, resize and size report of every terminal holds the sizing lock, and a PTY resize runs under it. The S1 census measured a full 300 ms wait on each (hosted run 41648095321: 300.2 and 300.3 ms). The first test holds each lock 300 ms (with an old-path control each). The second keeps the fix honest: two views of one terminal, the default policy; the other view's focus click takes the grid, and the next key from the typing view must take it back, on both arms. Red on this commit: the first test (the stage does not change these paths yet). --- .../cmux-tui-core/src/mux/terminal_sizing.rs | 7 +++ .../cmux-tui-core/src/server/activity.rs | 13 ++++ .../src/server/tests/rds_input_lane.rs | 61 +++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs index 1f771b53d6d2..1229dc6cafa5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs @@ -379,6 +379,13 @@ impl Mux { /// tests about latest-activity semantics (`latest` is also the default; the pin keeps /// such a test independent of the default). #[cfg(test)] + /// Runs `hold` while this thread holds the client sizing lock (tests: a person's key must not wait for it). + #[cfg(test)] + pub(crate) fn hold_client_sizing_for_test(&self, hold: &dyn Fn()) { + let _held = self.client_sizing.lock().unwrap_or_else(PoisonError::into_inner); + hold(); + } + pub(crate) fn pin_latest_size_policy_for_test(&self, surface: SurfaceId) { let workspace = self.surface_workspace(surface).expect("surface has a workspace"); self.set_workspace_size_policy( diff --git a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs index 2a478abbb1d4..d8c66bebfb22 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs @@ -55,6 +55,19 @@ impl ActivityStream { self.update(|state| state.agent_action_ms = Some(now_ms())); } + /// Runs `hold` while this thread holds the stream's lock (tests: a person's key must not wait for it). + #[cfg(test)] + pub(crate) fn hold_for_test(&self, hold: &dyn Fn()) { + let _held = self.shared.0.lock().unwrap_or_else(PoisonError::into_inner); + hold(); + } + + /// The last person's input time, in Unix milliseconds (tests). + #[cfg(test)] + pub(crate) fn user_input_ms_for_test(&self) -> Option { + self.shared.0.lock().unwrap_or_else(PoisonError::into_inner).user_input_ms + } + /// Mux events that change the counts or mean an agent acted. pub(crate) fn observe(&self, event: &MuxEvent) { match event { diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index af41bd24a238..2ba39f279a9a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -168,6 +168,14 @@ fn hold_creating_inputs(mux: &Arc, hold: &dyn Fn()) { mux.hold_creating_inputs_for_test(hold); } +fn hold_client_sizing(mux: &Arc, hold: &dyn Fn()) { + mux.hold_client_sizing_for_test(hold); +} + +fn hold_activity(mux: &Arc, hold: &dyn Fn()) { + mux.activity.hold_for_test(hold); +} + /// A terminal another creation names, still being created (`terminal-input-while-creating-v1`). const CREATING_TERMINAL: &str = "000000000000000000000000c4ea7e01"; /// A terminal nobody creates: a reader asks about it under the creating-input lock. @@ -331,6 +339,59 @@ fn a_key_waits_for_no_creating_input_lock() { ]); } +/// Stage S1 step 5: after a key, the terminal's sizing notes activity (the newest view sets the grid under the +/// default policy) and the VM activity stream notes a person's input. When the key's view is already the newest and +/// the grid is settled, the note changes nothing, so the key reads that from the terminal with no lock; the input +/// time is an atomic. Every attach, detach, resize and size report of every terminal holds the sizing lock, and a +/// PTY resize runs under it. +#[test] +fn a_key_waits_for_neither_the_sizing_nor_the_activity_lock() { + let sizing = "client sizing (Mux.client_sizing)"; + let activity = "VM activity stream (ActivityStream)"; + assert_old_path_waits_for(sizing, hold_client_sizing); + assert_old_path_waits_for(activity, hold_activity); + assert_keys_wait_for_none(&[ + (sizing, hold_client_sizing as Holder), + (activity, hold_activity as Holder), + ]); +} + +/// The skipped note must never keep the grid from a view: under the default policy (the newest view sets the grid), +/// another view's focus click takes the grid, and the next key from the first view takes it back, on both arms. +#[test] +fn after_another_views_click_a_key_takes_the_grid_back() { + for lane in [false, true] { + let first = TypingConnection::open_with(lane); + let mux = first.mux.clone(); + let surface = first.surface; + mux.pin_latest_size_policy_for_test(surface); + let (second_writer, _second_outbound) = captured_writer(); + let second = mux.control_clients.register(ClientTransport::Unix, second_writer.clone()); + attach_test_view(&mux, second, surface, &second_writer); + mux.resize_surface_for_client(surface, first.client, 100, 30).unwrap(); + mux.resize_surface_for_client(surface, second, 60, 20).unwrap(); + + first.submit("first-key", b'a'); + assert!(first.arrival(b'a').is_some(), "lane {lane}: the first key never arrived"); + first.settle(&["first-key"]); + let grid = + |mux: &Mux| mux.terminal_size_state(surface).map(|state| (state.cols, state.rows)); + assert_eq!(grid(&mux), Some((100, 30)), "lane {lane}: the typing view sets the grid"); + + assert_eq!(mux.note_terminal_activity(surface, second, None), Some(true), "lane {lane}"); + assert_eq!(grid(&mux), Some((60, 20)), "lane {lane}: the clicked view takes the grid"); + + first.submit("second-key", b'b'); + assert!(first.arrival(b'b').is_some(), "lane {lane}: the second key never arrived"); + first.settle(&["second-key"]); + assert_eq!(grid(&mux), Some((100, 30)), "lane {lane}: the key took the grid back"); + assert!(mux.activity.user_input_ms_for_test().is_some(), "lane {lane}: the input time"); + + disconnect_client(&mux, second, false); + first.close(); + } +} + /// #31940 and TYPING FIX 2 took these two off the key; this keeps them off on the stage's new path. #[test] fn a_key_waits_for_neither_the_session_nor_the_workspace_registry_lock() { From 0fb7c43a81042d32ead394b8e80d510597ac77c7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:32:12 +0000 Subject: [PATCH 19/42] fix(cmux-tui-util): a test dereferences the published u64 once (E0614 in bb0265f43d5) --- cmux-tui/crates/cmux-tui-util/src/published.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-util/src/published.rs b/cmux-tui/crates/cmux-tui-util/src/published.rs index 3f05dc7fdd79..8ee591304a6c 100644 --- a/cmux-tui/crates/cmux-tui-util/src/published.rs +++ b/cmux-tui/crates/cmux-tui-util/src/published.rs @@ -279,7 +279,7 @@ mod tests { let published = Published::new(Counter::default(), project, false); assert!(!published.publishing()); published.lock().unwrap_or_else(|poison| poison.into_inner()).value = 3; - assert_eq!(**published.load_full_for_test(), 0, "a change was published while off"); + assert_eq!(*published.load_full_for_test(), 0, "a change was published while off"); published.set_publishing_for_test(true); assert!(published.publishing()); assert_eq!(**published.load().unwrap_or_else(|poison| poison.into_inner()), 3); From 7fad48f75ff7ef251868217692e0a4446ddbf608 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:00:01 +0000 Subject: [PATCH 20/42] fix(cmux-tui-core): a key skips its sizing note when its view is the settled newest; the person's input time is an atomic (R-DAEMON-STATE S1 step 5) On the input lane (CMUX_TUI_RDS_INPUT, per mux, default off): - Each terminal's sizing entry publishes, on the terminal's runtime (shared by every placement), the view a key may skip its sizing note for: the sole newest participant, a client's own view, only while the entry is settled (no grid hold, no handheld with a viewport, and the engine holds the grid or the applied grid and participants are its decision). Noting activity for that view changes no decision under any policy, and the size state carries no activity, so the key compares (client, placement) with one atomic load and takes no sizing lock. - The publication is automatic: Mux::client_sizing is a SizingMutex whose guard republishes, on drop and under the lock, each entry its holder read or wrote (SizingEntries marks them; an apply changes Cells through a shared borrow, so reads count). A removed entry clears its slot; a change a panic cuts short publishes nothing. - Another view's activity makes that view the newest and republishes, so the next key from the first view takes the slow path once and takes the grid back (the second test pins this on both arms). - ActivityStream::note_user_input stores the time in an atomic; it takes the stream lock only while a subscriber exists, once per event the worker sends (a wake_pending flag the worker clears before it reads the time). - TerminalSizingEngine::sole_latest (cmux-terminal-sizing) with a unit test; AppliedSeat::of shares the seat rule between the apply and the settled check; set_workspace_size_policy reads its state before the note (no index). Off, every key takes the sizing and activity locks as on the base; the marks and the guard check one bool. --- .../crates/cmux-terminal-sizing/src/lib.rs | 44 ++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 6 +- .../src/mux/client_sizing_state.rs | 8 +- .../crates/cmux-tui-core/src/mux/construct.rs | 7 +- .../cmux-tui-core/src/mux/sizing_fast_path.rs | 237 ++++++++++++++++++ .../cmux-tui-core/src/mux/terminal_sizing.rs | 18 +- .../src/mux/terminal_sizing/grid_hold.rs | 35 ++- .../cmux-tui-core/src/server/activity.rs | 98 ++++++-- cmux-tui/crates/cmux-tui-core/src/surface.rs | 21 ++ .../crates/cmux-tui-core/src/surface/adopt.rs | 1 + .../crates/cmux-tui-core/src/surface/spawn.rs | 1 + .../src/surface/terminal_runtime.rs | 3 + .../cmux-tui-core/src/surface/test_spawn.rs | 1 + 13 files changed, 436 insertions(+), 44 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs diff --git a/cmux-tui/crates/cmux-terminal-sizing/src/lib.rs b/cmux-tui/crates/cmux-terminal-sizing/src/lib.rs index 166c74fcafd6..9ed90abbb6b1 100644 --- a/cmux-tui/crates/cmux-terminal-sizing/src/lib.rs +++ b/cmux-tui/crates/cmux-terminal-sizing/src/lib.rs @@ -453,6 +453,27 @@ impl TerminalSizingEngine { self.entries.iter().map(|entry| entry.participant.id.as_str()) } + /// The participant whose last activity is later than every other participant's, if one is. Noting activity + /// for it changes no decision: under every policy the newest of any set of views that holds it is still it, + /// and the newest of a set without it does not move. `None` while two views tie (handheld attaches, activity 0). + pub fn sole_latest(&self) -> Option<&str> { + let mut latest: Option<&Entry> = None; + let mut tied = false; + for entry in &self.entries { + match latest { + Some(current) if entry.activity < current.activity => {} + Some(current) if entry.activity == current.activity => tied = true, + _ => { + latest = Some(entry); + tied = false; + } + } + } + latest + .filter(|entry| !tied && entry.activity > 0) + .map(|entry| entry.participant.id.as_str()) + } + pub fn is_empty(&self) -> bool { self.entries.is_empty() } @@ -617,6 +638,29 @@ mod tests { ) } + #[test] + fn the_sole_latest_participant_is_the_one_that_acted_last_and_none_on_a_tie() { + let mut engine = TerminalSizingEngine::new( + TerminalGridSize::new(80, 24), + TerminalSizingPolicy::default(), + ); + assert_eq!(engine.sole_latest(), None, "no participant"); + let view = |id: &str, device_kind| TerminalSizingParticipant { + id: id.to_owned(), + device_kind, + ..TerminalSizingParticipant::default() + }; + engine.attach(view("phone-a", TerminalDeviceKind::Iphone)); + engine.attach(view("phone-b", TerminalDeviceKind::Iphone)); + assert_eq!(engine.sole_latest(), None, "two handheld attaches tie at activity 0"); + engine.attach(view("mac", TerminalDeviceKind::Mac)); + assert_eq!(engine.sole_latest(), Some("mac")); + engine.note_activity("phone-a"); + assert_eq!(engine.sole_latest(), Some("phone-a")); + engine.detach("phone-a"); + assert_eq!(engine.sole_latest(), Some("mac")); + } + #[test] fn sizing_policy_replays_shared_fixture_corpus() { let corpus: Value = serde_json::from_str(FIXTURES).unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a6d5d25327e0..2e7cf9ce3774 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -95,6 +95,8 @@ use cell_pixel_state::apply_cell_pixel_size_until; use cell_pixel_state::cell_pixel_retry_delay; use cell_pixel_state::validate_cell_pixel_convergence; mod client_sizing_state; +/// A key's sizing note without the sizing lock (R-DAEMON-STATE stage S1). +pub(crate) mod sizing_fast_path; use client_sizing_state::AppliedClientSize; use client_sizing_state::ClientResizeRequest; pub(crate) use client_sizing_state::ClientSizeRollback; @@ -535,7 +537,7 @@ pub struct Mux { workspace_lifecycles: Mutex>>>, pending_workspace_surfaces: Mutex>, client_sizing_lifecycle: Mutex<()>, - client_sizing: Mutex, + client_sizing: sizing_fast_path::SizingMutex, /// Per-client focus memory (client-focus-v1): the most recent focus each /// client id reported, so a reconnecting client restores its own view /// instead of the shared session focus. In-memory and bounded; a mux @@ -958,6 +960,8 @@ impl Mux { self.control_clients.set_input_lane_for_test(on); self.remote_relay().peers.set_publishing_for_test(on); self.surface_operation_admission.set_lock_free_for_test(on); + self.client_sizing.set_publishing_for_test(on); + self.activity.set_lock_free_for_test(on); } #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/client_sizing_state.rs b/cmux-tui/crates/cmux-tui-core/src/mux/client_sizing_state.rs index ef24cec39e35..3fd9aaec56e9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/client_sizing_state.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/client_sizing_state.rs @@ -84,6 +84,9 @@ pub(super) struct TerminalSizingEntry { /// After a phone or tablet departs, the grid does not grow past this size until the /// instant passes (`DEPARTED_GRID_HOLD`); it may shrink. pub(super) hold: std::cell::Cell>, + /// Where this terminal's runtime publishes the view a key may skip its sizing note for (stage S1); `None` when + /// no surface of the runtime was found when the entry was made (its keys always take the slow path). + pub(super) fast_path_slot: Option, } /// Which connection and placement one engine participant belongs to. @@ -123,8 +126,9 @@ pub(super) struct ClientSizingState { pub(super) next_size_order: u64, pub(super) policies: HashMap, pub(super) terminal_runtime_by_placement: HashMap, - /// Shared sizing engines keyed by terminal runtime id. - pub(super) terminal_sizing: HashMap, + /// Shared sizing engines keyed by terminal runtime id; each access marks its entry, which the sizing guard then + /// republishes (`sizing_fast_path`). + pub(super) terminal_sizing: sizing_fast_path::SizingEntries, /// Per-terminal policy overrides keyed by terminal runtime id. pub(super) terminal_size_policies: HashMap, /// Workspace default policies for terminals without an override. diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs index a4543472bdfb..7577fa04be93 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/construct.rs @@ -232,7 +232,10 @@ impl Mux { workspace_lifecycles: Mutex::new(HashMap::new()), pending_workspace_surfaces: Mutex::new(HashMap::new()), client_sizing_lifecycle: Mutex::new(()), - client_sizing: Mutex::new(ClientSizingState::default()), + client_sizing: sizing_fast_path::SizingMutex::new( + ClientSizingState::default(), + lane_on, + ), client_focus_memory: Mutex::new(Vec::new()), last_reported_focus: Mutex::new(None), conversations: crate::conversation_store::ConversationHost { @@ -422,7 +425,7 @@ impl Mux { agent_transfers: Default::default(), #[cfg(unix)] access_requests: Default::default(), - activity: Default::default(), + activity: crate::server::activity::ActivityStream::new(lane_on), idle_close: Mutex::new(idle_close::IdleCloseTracker::default()), idle_close_waker: Mutex::new(None), pty_reclaim_forced: AtomicBool::new(false), diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs b/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs new file mode 100644 index 000000000000..080c003347fe --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs @@ -0,0 +1,237 @@ +//! A key's sizing note without the sizing lock (R-DAEMON-STATE stage S1 step 5, +//! handoff/rewrites/R-DAEMON-STATE/DESIGN.md section 3). +//! +//! After its PTY write a key notes activity in its terminal's sizing: under the default policy the newest view sets +//! the grid. The note takes the daemon-wide sizing lock, which every attach, detach, resize and size report of every +//! terminal holds (a PTY resize runs under it). The note changes nothing when the key's view is already the only +//! newest participant and the grid is settled (nothing for the next apply to do): a newer activity of the newest +//! participant moves no decision under any policy, and the published size state carries no activity values. +//! +//! So each sizing entry publishes, on its terminal's runtime (the object every view of the terminal holds), the view +//! a key may skip the note for: `(client, placement)` of the entry's sole newest member, only while the entry is +//! settled, else nothing. The publication is automatic: every read or write of an entry goes through +//! [`SizingEntries`], which marks the entry, and the sizing guard republishes the marked entries when it drops, under +//! the lock, so the publications follow the changes (the `Cell`s an apply changes through a shared borrow included). +//! A key compares its own `(client, placement)` with one atomic load. Another view's activity makes that view the +//! newest and republishes; the next key from the first view then takes the slow path once and takes the grid back. + +use std::collections::HashMap; +use std::ops::{Deref, DerefMut}; +use std::sync::{Arc, LockResult, Mutex, MutexGuard, PoisonError}; + +use arc_swap::ArcSwapOption; + +use super::terminal_sizing::AppliedSeat; +use super::{ClientSizingState, TerminalSizingEntry}; +use crate::SurfaceId; +use crate::sizing_policy::TerminalSizingReason; + +/// The view a key may skip its sizing note for: a client's own view of one placement. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SizingFastPath { + pub(crate) client: u64, + pub(crate) placement: SurfaceId, +} + +/// Where a terminal runtime publishes its [`SizingFastPath`]; every view (placement) of the runtime shares it. +pub(crate) type SizingFastPathSlot = Arc>; + +/// The sizing entries by terminal runtime. Every access marks the entry (the guard republishes it), except a +/// lookup of keys or presence, which can change nothing. +#[derive(Default)] +pub(in crate::mux) struct SizingEntries { + map: HashMap, + /// Entries read or written since the guard last published; only while publishing. + touched: std::cell::RefCell>, + publishing: bool, +} + +impl SizingEntries { + fn mark(&self, runtime: SurfaceId) { + if self.publishing { + self.touched.borrow_mut().push(runtime); + } + } + + pub(in crate::mux) fn get(&self, runtime: &SurfaceId) -> Option<&TerminalSizingEntry> { + self.mark(*runtime); + self.map.get(runtime) + } + + pub(in crate::mux) fn get_mut( + &mut self, + runtime: &SurfaceId, + ) -> Option<&mut TerminalSizingEntry> { + self.mark(*runtime); + self.map.get_mut(runtime) + } + + pub(in crate::mux) fn insert( + &mut self, + runtime: SurfaceId, + entry: TerminalSizingEntry, + ) -> Option { + self.mark(runtime); + self.map.insert(runtime, entry) + } + + /// Removes the entry; a removed entry publishes nothing, so its terminal's keys take the slow path. + pub(in crate::mux) fn remove(&mut self, runtime: &SurfaceId) -> Option { + let removed = self.map.remove(runtime); + if self.publishing + && let Some(slot) = removed.as_ref().and_then(|entry| entry.fast_path_slot.as_ref()) + { + slot.store(None); + } + removed + } + + pub(in crate::mux) fn contains_key(&self, runtime: &SurfaceId) -> bool { + self.map.contains_key(runtime) + } + + pub(in crate::mux) fn keys(&self) -> impl Iterator { + self.map.keys() + } + + pub(in crate::mux) fn values(&self) -> impl Iterator { + self.map.iter().map(|(runtime, entry)| { + self.mark(*runtime); + entry + }) + } + + pub(in crate::mux) fn iter(&self) -> impl Iterator { + self.map.iter().inspect(|(runtime, _)| self.mark(**runtime)) + } + + /// Republishes every marked entry's fast path, or clears it when the change that marked it was cut short. + fn publish_marked(&self, cut_short: bool) { + let mut touched = std::mem::take(&mut *self.touched.borrow_mut()); + touched.sort_unstable(); + touched.dedup(); + for runtime in touched { + let Some(entry) = self.map.get(&runtime) else { continue }; + let Some(slot) = entry.fast_path_slot.as_ref() else { continue }; + let published = if cut_short { None } else { entry.fast_path().map(Arc::new) }; + slot.store(published); + } + } + + /// Turns publishing on or off (tests); every entry publishes, or clears, at once. + #[cfg(test)] + fn set_publishing(&mut self, on: bool) { + self.publishing = on; + for entry in self.map.values() { + if let Some(slot) = entry.fast_path_slot.as_ref() { + slot.store(if on { entry.fast_path().map(Arc::new) } else { None }); + } + } + self.touched.borrow_mut().clear(); + } +} + +impl TerminalSizingEntry { + /// The view a key may skip its note for: the sole newest member, a client's own view, while nothing waits for + /// the next apply. Settled: no grid hold, no handheld that reported a size (a handheld caps or holds the grid), + /// and, unless the engine holds the grid (no view counts, so an apply does nothing), the grid it applied is its + /// decision and the participants are the ones it applied for. + fn fast_path(&self) -> Option { + if self.hold.get().is_some() { + return None; + } + let sized_handheld = self.engine.participant_ids().any(|id| { + self.engine.participant(id).is_some_and(|participant| { + participant.device_kind.is_handheld() && participant.viewport.is_some() + }) + }); + if sized_handheld { + return None; + } + let state = self.engine.state(); + if state.reason != TerminalSizingReason::Held + && (self.applied.get() != Some((state.cols, state.rows)) || !self.applied_for_present()) + { + return None; + } + let newest = self.engine.sole_latest()?; + let member = self.members.get(newest).filter(|member| member.view.is_none())?; + Some(SizingFastPath { client: member.client, placement: member.placement }) + } + + /// Whether the last apply saw the participants there are now (else the next apply notices an arrival or a + /// departure); compared in place, with no copy. + fn applied_for_present(&self) -> bool { + let applied = self.applied_participants.borrow(); + applied.len() == self.engine.participant_ids().count() + && self + .engine + .participant_ids() + .all(|id| applied.get(id) == Some(&AppliedSeat::of(self.engine.participant(id)))) + } +} + +/// `Mux::client_sizing`: the sizing state behind its mutex, whose guard republishes the entries it touched. Whether +/// it publishes is the mux's stage S1 switch, kept with the entries under the lock. +pub(in crate::mux) struct SizingMutex { + value: Mutex, +} + +impl SizingMutex { + pub(in crate::mux) fn new(mut state: ClientSizingState, publishing: bool) -> Self { + state.terminal_sizing.publishing = publishing; + Self { value: Mutex::new(state) } + } + + pub(in crate::mux) fn lock(&self) -> LockResult> { + match self.value.lock() { + Ok(guard) => Ok(SizingGuard::new(guard)), + Err(poison) => Err(PoisonError::new(SizingGuard::new(poison.into_inner()))), + } + } + + /// Turns stage S1 on or off for the sizing (tests). + #[cfg(test)] + pub(in crate::mux) fn set_publishing_for_test(&self, on: bool) { + let mut guard = self.value.lock().unwrap_or_else(PoisonError::into_inner); + guard.terminal_sizing.set_publishing(on); + } +} + +/// A held [`SizingMutex`]. When it drops it republishes the entries it touched, before the mutex is released. +pub(in crate::mux) struct SizingGuard<'a> { + guard: MutexGuard<'a, ClientSizingState>, + /// Whether this thread was already unwinding when it took the lock, as `std::sync::MutexGuard` records it. + panicking_at_lock: bool, +} + +impl<'a> SizingGuard<'a> { + fn new(guard: MutexGuard<'a, ClientSizingState>) -> Self { + Self { guard, panicking_at_lock: std::thread::panicking() } + } +} + +impl Deref for SizingGuard<'_> { + type Target = ClientSizingState; + + fn deref(&self) -> &ClientSizingState { + &self.guard + } +} + +impl DerefMut for SizingGuard<'_> { + fn deref_mut(&mut self) -> &mut ClientSizingState { + &mut self.guard + } +} + +impl Drop for SizingGuard<'_> { + fn drop(&mut self) { + if self.guard.terminal_sizing.publishing { + // A panic that began while this guard held the lock cut its change short: the marked entries publish + // nothing (their keys take the slow path, which meets the poisoned lock as the old path does). + let cut_short = std::thread::panicking() && !self.panicking_at_lock; + self.guard.terminal_sizing.publish_marked(cut_short); + } + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs index 1229dc6cafa5..592f038a7acb 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs @@ -46,6 +46,12 @@ impl Mux { /// Takes the terminal the input was written to, not its id: a key's turn never looks it up through the session /// lock, which automation's lists, snapshots and durable mutations hold (TYPING UNDER LOAD F4). pub(crate) fn note_terminal_input(&self, surface: &Surface, client: u64) { + // The key's view is the terminal's settled newest: the note would change nothing, so the key takes no sizing + // lock (R-DAEMON-STATE stage S1, mux/sizing_fast_path.rs). + if self.control_clients.input_lane.on() && surface.skips_sizing_note(client) { + self.activity.note_user_input(); + return; + } if self.note_terminal_activity_of(surface, client, None).is_some() { self.activity.note_user_input(); } @@ -370,8 +376,9 @@ impl Mux { let resolved = mux.resolved_size_policy(sizing, runtime, surface); let entry = mux.terminal_sizing_entry(sizing, runtime, surface); let changed = entry.engine.set_policy(resolved); + let state = entry.engine.state().clone(); sizing.note_size_state(runtime, changed); - sizing.terminal_sizing[&runtime].engine.state().clone() + state })) } @@ -527,10 +534,10 @@ impl Mux { placement: SurfaceId, ) -> &'a mut TerminalSizingEntry { if !sizing.terminal_sizing.contains_key(&runtime) { - let (cols, rows) = self - .surface(placement) - .or_else(|| self.surface(runtime)) - .map_or((80, 24), |surface| surface.size()); + let surface = self.surface(placement).or_else(|| self.surface(runtime)); + let (cols, rows) = surface.as_ref().map_or((80, 24), |surface| surface.size()); + let fast_path_slot = + surface.as_ref().and_then(|surface| surface.sizing_fast_path_slot()); let policy = self.resolved_size_policy(sizing, runtime, placement); sizing.terminal_sizing.insert( runtime, @@ -541,6 +548,7 @@ impl Mux { applied: std::cell::Cell::new(None), applied_participants: std::cell::RefCell::new(std::collections::BTreeMap::new()), hold: std::cell::Cell::new(None), + fast_path_slot, }, ); } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing/grid_hold.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing/grid_hold.rs index aefa45768ee6..aef7dfb74fe1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing/grid_hold.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing/grid_hold.rs @@ -20,19 +20,7 @@ impl Mux { decided: (u16, u16), ) -> (u16, u16) { let now = Instant::now(); - let present = entry - .engine - .participant_ids() - .map(|id| { - let participant = entry.engine.participant(id); - let seat = AppliedSeat { - handheld: participant - .is_some_and(|participant| participant.device_kind.is_handheld()), - sized: participant.is_some_and(|participant| participant.viewport.is_some()), - }; - (id.to_owned(), seat) - }) - .collect::>(); + let present = AppliedSeat::present(entry); let (departed, returned) = { let previous = entry.applied_participants.borrow(); let departed = @@ -243,6 +231,27 @@ pub(in crate::mux) struct AppliedSeat { sized: bool, } +impl AppliedSeat { + /// One participant as an apply records it. + pub(in crate::mux) fn of(participant: Option<&TerminalSizingParticipant>) -> Self { + AppliedSeat { + handheld: participant.is_some_and(|participant| participant.device_kind.is_handheld()), + sized: participant.is_some_and(|participant| participant.viewport.is_some()), + } + } + + /// Every participant of `entry` now, as an apply records them. + pub(in crate::mux) fn present( + entry: &TerminalSizingEntry, + ) -> std::collections::BTreeMap { + entry + .engine + .participant_ids() + .map(|id| (id.to_owned(), Self::of(entry.engine.participant(id)))) + .collect() + } +} + /// `Mux::terminal_grid_hold_snapshot`. pub(in crate::mux) struct TerminalGridHoldSnapshot { participants: std::collections::BTreeMap, diff --git a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs index d8c66bebfb22..7a8d44cb04bd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs @@ -8,6 +8,7 @@ //! timeout only to keep the one-second spacing. It exits when the last subscriber leaves. use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex, PoisonError, Weak}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -35,9 +36,37 @@ struct State { mux: Weak, } +/// The stream's state behind its lock, the worker's wake, and the two facts a person's key writes with no lock +/// (R-DAEMON-STATE stage S1). +#[derive(Default)] +struct Shared { + state: Mutex, + wake: Condvar, + /// A person's last input time in Unix milliseconds (0: none) on the lock-free arm; `State::user_input_ms` stays + /// `None` there. + user_input_ms: AtomicU64, + /// Whether the stream has subscribers; written under `state`'s lock. + subscribed: AtomicBool, + /// A key already asked the worker for an event that is not out yet; cleared by the worker under the lock. + wake_pending: AtomicBool, +} + +impl Shared { + /// A person's last input time from either arm. + fn user_input_ms(&self, state: &State) -> Option { + let atomic = Some(self.user_input_ms.load(Ordering::SeqCst)).filter(|ms| *ms != 0); + match (state.user_input_ms, atomic) { + (Some(locked), Some(atomic)) => Some(locked.max(atomic)), + (locked, atomic) => locked.or(atomic), + } + } +} + #[derive(Default)] pub(crate) struct ActivityStream { - shared: Arc<(Mutex, Condvar)>, + shared: Arc, + /// The mux's stage S1 switch: a person's key notes its input time with no lock. + lock_free: AtomicBool, } fn now_ms() -> u64 { @@ -45,9 +74,31 @@ fn now_ms() -> u64 { } impl ActivityStream { - /// Input from a person's attached client (callers filter unattached one-shot sends). + /// A stream whose person input notes take no lock when `lock_free` (the mux's stage S1 switch). + pub(crate) fn new(lock_free: bool) -> Self { + Self { shared: Arc::default(), lock_free: AtomicBool::new(lock_free) } + } + + /// Turns stage S1 on or off (tests). + #[cfg(test)] + pub(crate) fn set_lock_free_for_test(&self, on: bool) { + self.lock_free.store(on, Ordering::Relaxed); + } + + /// Input from a person's attached client (callers filter unattached one-shot sends). On the lock-free arm the + /// time is an atomic; only while a subscriber exists does a key take the lock, once per event the worker sends. pub(crate) fn note_user_input(&self) { - self.update(|state| state.user_input_ms = Some(now_ms())); + if !self.lock_free.load(Ordering::Relaxed) { + self.update(|state| state.user_input_ms = Some(now_ms())); + return; + } + // SeqCst: either this key sees the subscriber, or the subscriber's first snapshot sees this time. + self.shared.user_input_ms.store(now_ms(), Ordering::SeqCst); + if self.shared.subscribed.load(Ordering::SeqCst) + && !self.shared.wake_pending.swap(true, Ordering::SeqCst) + { + self.update(|_| {}); + } } /// An agent hook commit or an agent state report. @@ -58,14 +109,15 @@ impl ActivityStream { /// Runs `hold` while this thread holds the stream's lock (tests: a person's key must not wait for it). #[cfg(test)] pub(crate) fn hold_for_test(&self, hold: &dyn Fn()) { - let _held = self.shared.0.lock().unwrap_or_else(PoisonError::into_inner); + let _held = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); hold(); } /// The last person's input time, in Unix milliseconds (tests). #[cfg(test)] pub(crate) fn user_input_ms_for_test(&self) -> Option { - self.shared.0.lock().unwrap_or_else(PoisonError::into_inner).user_input_ms + let state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); + self.shared.user_input_ms(&state) } /// Mux events that change the counts or mean an agent acted. @@ -78,12 +130,11 @@ impl ActivityStream { } fn update(&self, change: impl FnOnce(&mut State)) { - let (lock, wake) = &*self.shared; - let mut state = lock.lock().unwrap_or_else(PoisonError::into_inner); + let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); change(&mut state); if !state.subscribers.is_empty() { state.dirty = true; - wake.notify_one(); + self.shared.wake.notify_one(); } } @@ -94,8 +145,7 @@ impl ActivityStream { writer: &MessageWriter, ) -> anyhow::Result { let times = { - let (lock, _) = &*self.shared; - let mut state = lock.lock().unwrap_or_else(PoisonError::into_inner); + let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); anyhow::ensure!( state.subscribers.len() < MAX_SUBSCRIBERS || state.subscribers.contains_key(&client), @@ -105,6 +155,9 @@ impl ActivityStream { state.subscribers.retain(|_, w| w.is_open()); state.mux = Arc::downgrade(mux); state.subscribers.insert(client, writer.clone()); + self.shared.subscribed.store(true, Ordering::SeqCst); + // A wake a key asked for before any subscriber existed is not pending for this one. + self.shared.wake_pending.store(false, Ordering::SeqCst); if !state.worker { state.worker = true; let shared = Arc::clone(&self.shared); @@ -112,16 +165,16 @@ impl ActivityStream { .name("cmux-activity".into()) .spawn(move || run_worker(shared))?; } - (state.user_input_ms, state.agent_action_ms) + (self.shared.user_input_ms(&state), state.agent_action_ms) }; Ok(json!({ "activity": snapshot(mux, times) })) } fn disconnect(&self, client: u64) { - let (lock, wake) = &*self.shared; - let mut state = lock.lock().unwrap_or_else(PoisonError::into_inner); + let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); if state.subscribers.remove(&client).is_some() && state.subscribers.is_empty() { - wake.notify_one(); + self.shared.subscribed.store(false, Ordering::SeqCst); + self.shared.wake.notify_one(); } } } @@ -193,12 +246,13 @@ fn snapshot(mux: &Mux, (user_input_ms, agent_action_ms): (Option, Option, (Option, Option), Weak); /// Blocks until a change is due; `None` when the last subscriber left. -fn next_due(shared: &(Mutex, Condvar)) -> Option { - let (lock, wake) = shared; - let mut state = lock.lock().unwrap_or_else(PoisonError::into_inner); +fn next_due(shared: &Shared) -> Option { + let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); loop { if state.subscribers.is_empty() { state.worker = false; + shared.subscribed.store(false, Ordering::SeqCst); + shared.wake_pending.store(false, Ordering::SeqCst); return None; } if state.dirty { @@ -208,18 +262,20 @@ fn next_due(shared: &(Mutex, Condvar)) -> Option { if wait.is_zero() { break; } - state = wake.wait_timeout(state, wait).unwrap_or_else(PoisonError::into_inner).0; + state = shared.wake.wait_timeout(state, wait).unwrap_or_else(PoisonError::into_inner).0; continue; } - state = wake.wait(state).unwrap_or_else(PoisonError::into_inner); + state = shared.wake.wait(state).unwrap_or_else(PoisonError::into_inner); } state.dirty = false; + // Before the time is read: a key that finds this cleared asks for the next event. + shared.wake_pending.store(false, Ordering::SeqCst); state.last_emit = Some(Instant::now()); let writers = state.subscribers.iter().map(|(c, w)| (*c, w.clone())).collect(); - Some((writers, (state.user_input_ms, state.agent_action_ms), state.mux.clone())) + Some((writers, (shared.user_input_ms(&state), state.agent_action_ms), state.mux.clone())) } -fn run_worker(shared: Arc<(Mutex, Condvar)>) { +fn run_worker(shared: Arc) { while let Some((writers, times, mux)) = next_due(&shared) { let Some(mux) = mux.upgrade() else { break }; let event = json!({ "event": "activity-changed", "activity": snapshot(&mux, times) }); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 10ae2329901b..2d0ca280f2f4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -621,6 +621,26 @@ impl Surface { }))) } + /// Where this terminal's sizing publishes the view a key may skip its sizing note for (R-DAEMON-STATE stage S1); + /// every view of the terminal shares it. `None` for a browser surface. + pub(crate) fn sizing_fast_path_slot( + &self, + ) -> Option { + self.as_pty().map(|pty| pty.terminal.sizing_fast_path.clone()) + } + + /// Whether a key from `client` into this view may skip its sizing note: the terminal's sizing published this + /// view as its settled newest. One atomic load, no lock. + pub(crate) fn skips_sizing_note(&self, client: u64) -> bool { + self.as_pty().is_some_and(|pty| { + pty.terminal + .sizing_fast_path + .load() + .as_deref() + .is_some_and(|fast| fast.client == client && fast.placement == self.id) + }) + } + pub(crate) fn shares_terminal_runtime(&self, other: &Surface) -> bool { match (self, other) { (Surface::Pty(left), Surface::Pty(right)) => { @@ -849,6 +869,7 @@ impl Surface { taps: RankedMutex::new(LockRank::AttachTaps, "pty.taps", Vec::new()), attach_colors_pending: AtomicBool::new(false), attach_colors_force_pending: AtomicBool::new(false), + sizing_fast_path: Default::default(), snapshot_position: Default::default(), last_attach_colors: RankedMutex::new( LockRank::Leaf, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs b/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs index 4126d25a7775..66bd5b1f08d3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs @@ -288,6 +288,7 @@ impl Surface { taps: RankedMutex::new(LockRank::AttachTaps, "pty.taps", Vec::new()), attach_colors_pending: AtomicBool::new(false), attach_colors_force_pending: AtomicBool::new(false), + sizing_fast_path: Default::default(), snapshot_position: Default::default(), last_attach_colors: RankedMutex::new( LockRank::Leaf, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs b/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs index 13276d1aef6c..4d094c5c4b4a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs @@ -287,6 +287,7 @@ impl Surface { taps: RankedMutex::new(LockRank::AttachTaps, "pty.taps", Vec::new()), attach_colors_pending: AtomicBool::new(false), attach_colors_force_pending: AtomicBool::new(false), + sizing_fast_path: Default::default(), snapshot_position: Default::default(), last_attach_colors: RankedMutex::new( LockRank::Leaf, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index e60e7d96cf6b..7747ac0a2a0b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -307,6 +307,9 @@ pub struct PtyTerminalRuntime { pub(super) render: Arc>, pub(super) render_generation: AtomicU64, pub(super) frame_requests: frame_producer::FrameRequests, + /// The view a key may skip its sizing note for, published by the terminal's sizing entry (R-DAEMON-STATE stage + /// S1, `mux::sizing_fast_path`); shared by every view of the terminal. + pub(super) sizing_fast_path: crate::mux::sizing_fast_path::SizingFastPathSlot, #[cfg(test)] pub(super) frame_producer_before_upgrade: FrameProducerTestHook, } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs b/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs index 889be802dd17..e101986e8299 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs @@ -240,6 +240,7 @@ impl Surface { taps: RankedMutex::new(LockRank::AttachTaps, "pty.taps", Vec::new()), attach_colors_pending: AtomicBool::new(false), attach_colors_force_pending: AtomicBool::new(false), + sizing_fast_path: Default::default(), snapshot_position: Default::default(), last_attach_colors: RankedMutex::new( LockRank::Leaf, From cd9a6dc912923205823da67d11ae33a53e0136a6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:29:38 +0000 Subject: [PATCH 21/42] fix(cmux-tui-core): the sizing test hook went in above another test hook's cfg(test) (dead code and 2 expect hits in production) dfe81bff4d8 put hold_client_sizing_for_test between pin_latest_size_policy_for_test's doc and its #[cfg(test)], so pin_latest_size_policy_for_test compiled into production: clippy -D warnings dead_code, and the panic ratchet's cmux-tui-core expect_used 175 > 173 (hosted lint job 126202464363). --- cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs index 592f038a7acb..2f317d4cb082 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs @@ -382,10 +382,6 @@ impl Mux { })) } - /// Pins the `latest` policy on the workspace that shows `surface`, for - /// tests about latest-activity semantics (`latest` is also the default; the pin keeps - /// such a test independent of the default). - #[cfg(test)] /// Runs `hold` while this thread holds the client sizing lock (tests: a person's key must not wait for it). #[cfg(test)] pub(crate) fn hold_client_sizing_for_test(&self, hold: &dyn Fn()) { @@ -393,6 +389,10 @@ impl Mux { hold(); } + /// Pins the `latest` policy on the workspace that shows `surface`, for + /// tests about latest-activity semantics (`latest` is also the default; the pin keeps + /// such a test independent of the default). + #[cfg(test)] pub(crate) fn pin_latest_size_policy_for_test(&self, surface: SurfaceId) { let workspace = self.surface_workspace(surface).expect("surface has a workspace"); self.set_workspace_size_policy( From 19dd118885de84f16514b20147330d91ac8d1be9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:35:42 +0000 Subject: [PATCH 22/42] fix(cmux-tui-core): the lane's surface read loads the routes before its poison check; two test doc comments on their items (#34171 review 2) Review comment 6615927915 (SHIP at 0fb7c43a810, notes): Mux::surface checked is_poisoned() and then loaded the routes; between the two another thread could poison the state and a third, recovering the poison, publish half-made tables. Poison never clears on the session state, so routes loaded before an unpoisoned check were published before any poisoning: load first, then check. Two doc comments in remote_relay/tests.rs sat above the wrong item. --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 11 +++++++---- .../cmux-tui-core/src/server/remote_relay/tests.rs | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a6d5d25327e0..e1685fa49f3f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -929,10 +929,13 @@ impl Mux { #[track_caller] pub fn surface(&self, id: SurfaceId) -> Option> { - // The routes of the last finished change, with no lock (R-DAEMON-STATE stage S1); a poisoned state takes the - // old read and fails as it does (a guard that recovers the poison may publish half-made tables). - if self.control_clients.input_lane.on() && !self.state.is_poisoned() { - return self.state.routes().surface(id); + // The last finished change's routes, no lock (R-DAEMON-STATE S1); loaded before the poison check (poison + // never clears), and a poisoned state takes the old read, which fails as it does. + if self.control_clients.input_lane.on() { + let routes = self.state.routes(); + if !self.state.is_poisoned() { + return routes.surface(id); + } } let state = self.state.lock().unwrap(); state.surfaces.get(&id).or_else(|| state.terminal_runtime_by_id(id)).cloned() diff --git a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs index c542b2694570..090dec91250c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/remote_relay/tests.rs @@ -955,7 +955,6 @@ fn a_remote_entry_connection_keeps_the_remote_path_after_disconnect() { // Poisoned relay locks (lane 10 lock slice): every admission, binding and // revocation step fails closed and returns a typed error. -/// Poisons `mutex`: a panic while it is held. /// Poisons a published value: a panic while its lock is held. fn poison_published(value: &crate::published::Published) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { @@ -965,6 +964,7 @@ fn poison_published(value: &crate::published::Published) { assert!(value.is_poisoned()); } +/// Poisons `mutex`: a panic while it is held. fn poison(mutex: &Mutex) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let _held = mutex.lock().unwrap(); @@ -1035,8 +1035,6 @@ fn a_poisoned_peers_lock_gives_no_principal_and_no_local_trust() { } } -/// The same for the client registry, also on the input lane, whose readers -/// load the registry's published facts instead of its lock. /// A panic while the session state is held: with the lane on, the next surface read takes the old path and fails /// as it does, never routing by tables that a guard which recovered the poison may have published. No terminal: /// the mux has none, so nothing else locks the poisoned state. @@ -1053,6 +1051,8 @@ fn a_poisoned_session_state_never_routes_by_the_published_tables() { assert!(read.is_err(), "the lane read the published routes of a poisoned session state"); } +/// The same for the client registry, also on the input lane, whose readers +/// load the registry's published facts instead of its lock. #[test] fn a_poisoned_registry_lock_trusts_no_client_as_local() { for lane in [false, true] { From eaef70c313f981f4571ed6bbfe98bfcb372887de Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:03:54 +0000 Subject: [PATCH 23/42] test(cmux-tui-core): a key does not wait for another writer's blocked host socket (R-DAEMON-STATE S1 step 6 red) A key to a hosted terminal holds the terminal's runtime lock and its HostAttachment.writer across a blocking write to the host socket (surface/input.rs, attachment.rs send_host_frame). Another client's large paste to the same terminal holds both for its whole write, bounded only by the socket's send timeout, so the person's key waits for automation's work on the same terminal (DESIGN.md section 2.6.1, Appendix A.2 steps 20-22). The test pastes 8 MiB while the fake host reads nothing for 600 ms, then sends one key. A control first proves the old path waits (the key takes at least half the stall); the lane must return the key in < 100 ms, and the host must then read the paste before the key, and the key once. Red on this commit: the lane has no input port yet. --- .../cmux-tui-core/src/surface/tests/hosted.rs | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index e8e84f3d9cdf..7dc1f5ccd5e3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -88,6 +88,111 @@ fn hosted_receipted_input_requests_pipeline_through_surface_reader() { }); } +/// A hosted surface on a fake host connection (the host end is the second value), with the mux's stage S1 switch +/// `lane`. +#[cfg(unix)] +fn hosted_typing_surface( + name: &str, + lane: bool, +) -> (Arc, Arc, std::os::unix::net::UnixStream) { + let mux = Mux::new_for_test(name, SurfaceOptions::default()); + mux.set_input_lane_for_test(lane); + let workspace = mux.create_empty_workspace(None, None, None).unwrap(); + let (mut attachment, host) = crate::terminal_host_runtime::input_ack_surface_fixture(); + let terminal_id = attachment.record.terminal_id.clone(); + attachment.record.workspace_key = workspace.key.clone(); + mux.seed_launching_terminal_for_test(&terminal_id, &workspace.key).unwrap(); + let surface = Surface::spawn_hosted( + 1, + SurfaceOptions::default(), + Arc::downgrade(&mux), + HostedSurfaceLaunch { + adopted: false, + attachment, + kitty_reservation: None, + terminate_on_error: false, + defer_launch_activation: false, + lifetime: PtyLifetime::SessionOwned, + terminal_public_id: None, + resource_identity: None, + }, + ) + .unwrap(); + host.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + (mux, surface, host) +} + +/// How long the fake host reads nothing before it drains its socket. +#[cfg(unix)] +const HOST_STALL: Duration = Duration::from_millis(600); + +/// A key while another client's large paste to the same hosted terminal is in its socket write and the host reads +/// nothing for [`HOST_STALL`]: how long the key took to return, and the frame kinds the host then read, in order. +/// The host drains on its own thread after the stall, so the old path (the key waits for the paste) cannot hang. +#[cfg(unix)] +fn key_behind_a_blocked_paste(lane: bool) -> (Duration, Vec) { + let (_mux, surface, mut host) = hosted_typing_surface("hosted-input-port", lane); + // Far larger than the socket buffer: its write blocks while the host reads nothing. + let paste = vec![b'p'; 8 * 1024 * 1024]; + std::thread::scope(|scope| { + let reader = scope.spawn(move || { + std::thread::sleep(HOST_STALL); + let mut kinds = Vec::new(); + while kinds.last() != Some(&MessageKind::Input) { + let frame = crate::terminal_host_protocol::read_frame( + &mut host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) + .expect("the host reads the queued frames") + .expect("the host connection stays open"); + kinds.push(frame.kind); + } + kinds + }); + let pasting = scope.spawn(|| surface.write_paste(&paste)); + // Long enough for the paste's writer to be inside its blocked socket write, well inside the stall. + std::thread::sleep(Duration::from_millis(150)); + let started = Instant::now(); + surface.write_bytes(b"k").expect("the key was accepted"); + let took = started.elapsed(); + pasting.join().unwrap().expect("the paste was written"); + (took, reader.join().unwrap()) + }) +} + +/// R-DAEMON-STATE stage S1 step 6 (DESIGN.md section 2.6.1): a key to a hosted terminal never waits for another +/// writer's blocked write to the same terminal's host socket. On the old path the paste's write holds the +/// terminal's runtime lock and the host writer for the whole write (bounded only by the socket's send timeout), and +/// the key waits behind it. With the stage on, the key is handed to the terminal's own input port and returns at +/// once; the port writes in order, the paste first, once the host reads. +#[cfg(unix)] +#[test] +fn a_key_does_not_wait_for_another_writers_blocked_host_socket() { + let (old_took, old_kinds) = key_behind_a_blocked_paste(false); + assert!( + old_took >= HOST_STALL / 2, + "control: on the old path the key took {old_took:?} behind a paste blocked {HOST_STALL:?}; the hold does not \ + reach the old path, so the lane's test proves nothing" + ); + assert_eq!(old_kinds.last(), Some(&MessageKind::Input), "old path: the key reached the host"); + + let (took, kinds) = key_behind_a_blocked_paste(true); + assert!( + took < Duration::from_millis(100), + "the key waited {took:?} for another writer's blocked host socket" + ); + assert_eq!( + kinds.iter().filter(|kind| **kind == MessageKind::Input).count(), + 1, + "the key reached the host once: {kinds:?}" + ); + assert!( + kinds.iter().position(|kind| *kind == MessageKind::Paste) + < kinds.iter().position(|kind| *kind == MessageKind::Input), + "the paste, written first, reached the host before the key: {kinds:?}" + ); +} + #[cfg(unix)] #[test] fn hosted_stager_exposes_coupled_state_only_after_colors() { From 331fe4947d8f998a5baea498c71e08dca6ebb70b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:45:56 +0000 Subject: [PATCH 24/42] fix(cmux-tui-core): a person's key to a hosted terminal goes to the terminal's input port and never waits for a socket write (R-DAEMON-STATE S1 step 6) On the input lane (CMUX_TUI_RDS_INPUT, per mux, default off), interactive input to a hosted terminal (Surface::write_bytes, Surface::write_paste) goes to the terminal's own input port (surface/input_port.rs): one queue per terminal, drained in order by the terminal's writer thread (terminal-input-port, started on the first input, ended after 30 s idle). The request returns once the port took the bytes; it takes neither the terminal's runtime lock nor the host writer, so another writer of the same terminal (a large paste, receipted input, a cell-pixel change waiting for its acknowledgement) and a host that does not read its socket no longer hold the person's key or the connection's dispatcher. - The writer writes on the route of the current host connection (HostInputRoute: the connection's writer), which the terminal publishes whenever its runtime changes: the hosted and exited constructors, and PtyTerminalRuntime::replace_runtime, which the reconnect install and the host exit now use. So the writer never takes the runtime lock either. - A lost connection is retried as write_bytes did (cx-6so.49), on the port's thread: wait for the reader's next stream change, write to the new host, until HOST_INPUT_RECONNECT_WAIT. - A full port (32 MiB queued) refuses at once (terminal.input_busy, WouldBlock); a write larger than one host frame is refused as before; the exited final screen drops input as before. - Input the port took and then could not write is lost as a failed write was, with a log line: the request was already answered (the reply now means "accepted by the terminal's port", DESIGN.md 2.6.1). - Not in this step (S1.6b, before the default flip): the connection's other frames still write directly, so while the port is behind they can go before a key it still holds. Off, every write takes the path it took on the base; the route is published on both arms (one Arc per change). --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 7 + .../crates/cmux-tui-core/src/surface/adopt.rs | 4 + .../cmux-tui-core/src/surface/exit_state.rs | 2 +- .../src/surface/hosted_reader/reconnect.rs | 2 +- .../crates/cmux-tui-core/src/surface/input.rs | 36 ++- .../cmux-tui-core/src/surface/input_port.rs | 227 ++++++++++++++++++ .../crates/cmux-tui-core/src/surface/spawn.rs | 4 + .../src/surface/terminal_runtime.rs | 32 +++ .../cmux-tui-core/src/surface/test_spawn.rs | 4 + .../src/terminal_host_runtime.rs | 1 + .../shared/attachment.rs | 21 ++ 11 files changed, 335 insertions(+), 5 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 2d0ca280f2f4..7714f6d264b6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -61,6 +61,9 @@ use scrolling::{ }; #[cfg(any(unix, windows))] mod hosted_stager; +/// The terminal's own input port (R-DAEMON-STATE stage S1 step 6). +#[cfg(any(unix, windows))] +pub(crate) mod input_port; #[cfg(any(unix, windows))] use hosted_stager::{HostedFrameStager, HostedTransition}; #[cfg(test)] @@ -808,6 +811,10 @@ impl Surface { "pty.mouse_encoders", Box::new(mouse_encoders), ), + input_route: arc_swap::ArcSwapOption::new(Some(Arc::new( + input_port::InputRoute::Hosted(attachment.input_route()), + ))), + input_port: Default::default(), runtime: RankedMutex::new( LockRank::Runtime, "pty.runtime", diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs b/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs index 66bd5b1f08d3..08bef432a7e0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs @@ -229,6 +229,10 @@ impl Surface { "pty.mouse_encoders", Box::new(mouse_encoders), ), + input_route: arc_swap::ArcSwapOption::new(Some(Arc::new( + super::input_port::InputRoute::Exited, + ))), + input_port: Default::default(), runtime: RankedMutex::new( LockRank::Runtime, "pty.runtime", diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/exit_state.rs b/cmux-tui/crates/cmux-tui-core/src/surface/exit_state.rs index c9f88de31b10..cb7dde61f991 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/exit_state.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/exit_state.rs @@ -119,7 +119,7 @@ pub(super) fn mark_hosted_runtime_exited( if let PtyRuntime::Hosted(host) = &*runtime { host.disconnect(); } - *runtime = PtyRuntime::ExitedHosted; + pty.replace_runtime(&mut runtime, PtyRuntime::ExitedHosted); pty.supports_clear_history_key_fallback.store(false, Ordering::Release); drop(runtime); pty.finish_hosted_exit(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs index 69d16c794429..21a934155076 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs @@ -130,7 +130,7 @@ impl HostedReader { { replacement.keep_pty_custody(custody); } - *runtime = PtyRuntime::Hosted(Box::new(replacement)); + pty.replace_runtime(&mut runtime, PtyRuntime::Hosted(Box::new(replacement))); pty.supports_clear_history_key_fallback .store(supports_clear_history, Ordering::Release); true diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index fd0d4f0ab4ff..00de6f4ccb6d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -8,13 +8,14 @@ use super::*; /// /// The wait blocks its caller: one control connection's next request, or /// the TUI's input worker. It applies only while a host connection is lost, -/// and a replacement host usually installs within a second. -const HOST_INPUT_RECONNECT_WAIT: Duration = Duration::from_secs(5); +/// and a replacement host usually installs within a second. On the input lane +/// the terminal's input port waits instead (`surface::input_port`). +pub(super) const HOST_INPUT_RECONNECT_WAIT: Duration = Duration::from_secs(5); /// Whether a host send failed because the host connection is gone (the /// host died or its stream broke), not because the frame was refused. #[cfg(any(unix, windows))] -fn host_connection_lost(error: &std::io::Error) -> bool { +pub(super) fn host_connection_lost(error: &std::io::Error) -> bool { matches!( error.kind(), std::io::ErrorKind::BrokenPipe @@ -34,6 +35,10 @@ impl Surface { )); }; #[cfg(any(unix, windows))] + if let Some(result) = Self::enqueue_on_input_port(pty, MessageKind::Input, bytes) { + return result; + } + #[cfg(any(unix, windows))] if let Some(result) = Self::send_hosted_input(pty, MessageKind::Input, bytes) { return result; } @@ -56,6 +61,27 @@ impl Surface { } } + /// On the input lane (R-DAEMON-STATE stage S1 step 6): hands interactive input for a hosted terminal to the + /// terminal's input port and returns once the port took it; `None` when the lane is off or the terminal is not + /// hosted (the caller writes as before). The final screen of an exited host drops the input, as before. + #[cfg(any(unix, windows))] + fn enqueue_on_input_port( + pty: &PtySurface, + kind: MessageKind, + bytes: &[u8], + ) -> Option> { + if !pty.mux.upgrade().is_some_and(|mux| mux.control_clients.input_lane.on()) { + return None; + } + match pty.input_route.load().as_deref() { + None => None, + Some(super::input_port::InputRoute::Exited) => Some(Ok(())), + Some(super::input_port::InputRoute::Hosted(_)) => { + Some(pty.input_port.enqueue(&pty.terminal, kind, bytes).map_err(Into::into)) + } + } + } + /// Send input to a hosted terminal's host, across a lost host /// connection (cx-6so.49). A host that died or whose stream broke has a /// dead connection until the surface reader installs the reconnected or @@ -182,6 +208,10 @@ impl Surface { // Keep-on-exit terminals accept and drop paste input the same way // as keystrokes: the final screen is read-only, not broken. #[cfg(any(unix, windows))] + if let Some(result) = Self::enqueue_on_input_port(pty, MessageKind::Paste, bytes) { + return result; + } + #[cfg(any(unix, windows))] if let Some(result) = Self::send_hosted_input(pty, MessageKind::Paste, bytes) { return result; } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs new file mode 100644 index 000000000000..0523b3d7828a --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -0,0 +1,227 @@ +//! The terminal's own input port (R-DAEMON-STATE stage S1 step 6, handoff/rewrites/R-DAEMON-STATE/DESIGN.md +//! section 2.6.1). +//! +//! A person's key to a hosted terminal was written to the host socket on the request thread, under the terminal's +//! runtime lock and the host connection's writer. Another writer of the same terminal (a large paste, receipted +//! input, a cell-pixel change waiting for its acknowledgement) held both for its whole write, so the key waited +//! behind it, and a host that did not read its socket stalled the connection's dispatcher. +//! +//! With the stage switch on (`CMUX_TUI_RDS_INPUT`), interactive input to a hosted terminal (keys and pastes: +//! `Surface::write_bytes`, `Surface::write_paste`) goes to the terminal's port instead: one queue per terminal, +//! drained in order by the terminal's own writer thread. The request returns once the port took the bytes; it takes +//! neither the runtime lock nor the host writer. +//! +//! - The writer writes on the route of the terminal's current host connection, which the terminal publishes each +//! time the connection changes (`PtyTerminalRuntime::replace_runtime`), so it never takes the runtime lock either. +//! - A lost host connection is retried as `write_bytes` did (cx-6so.49): the writer waits for the reader's next +//! stream change and writes to the new host, until `HOST_INPUT_RECONNECT_WAIT`. +//! - A full port refuses at once with a typed error: nothing is dropped silently and nothing waits. +//! - Input the port took and then could not write (the host is gone and no new one came) is lost, as a failed write +//! was before; the request had already been answered, so the port logs it. +//! - Stage S1 step 6b, before the default flip: every other frame of the connection (viewer size, cell pixels, +//! receipted input, clear history, Kitty limits, clipboard replies, terminate) still writes directly, so while the +//! port is behind such a frame can go before a key the port still holds. + +use std::collections::VecDeque; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Condvar, Mutex, PoisonError, Weak}; +use std::time::{Duration, Instant}; + +use super::input::HOST_INPUT_RECONNECT_WAIT; +use super::{PtyTerminalRuntime, TerminalHostConnectionState}; +use crate::terminal_host_protocol::{MAX_FRAME_PAYLOAD, MessageKind}; +use crate::terminal_host_runtime::HostInputRoute; + +/// The most bytes one terminal's port holds; more is refused (`terminal.input_busy`). Two of the largest frames. +const INPUT_PORT_BYTE_CAPACITY: usize = 2 * MAX_FRAME_PAYLOAD; +/// A port writer with nothing to write for this long ends; the next input starts a new one. +const INPUT_PORT_IDLE_EXIT: Duration = Duration::from_secs(30); +/// The writer's stack: it formats and writes one frame at a time. +const INPUT_PORT_STACK_BYTES: usize = 256 * 1024; + +/// Where a hosted terminal's interactive input goes, published without the runtime lock. +pub(crate) enum InputRoute { + /// The current host connection. + Hosted(HostInputRoute), + /// The host exited and the terminal keeps its final screen: input is dropped, as `write_bytes` drops it. + Exited, +} + +/// Why the port did not take a write. +#[derive(Debug)] +pub(crate) enum InputRefused { + /// The port already holds [`INPUT_PORT_BYTE_CAPACITY`] bytes (retryable). + Busy, + /// One write larger than a host frame. + TooLarge, + /// No writer thread could start. + NoWriter(std::io::Error), +} + +impl From for std::io::Error { + fn from(refused: InputRefused) -> Self { + match refused { + InputRefused::Busy => std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "terminal.input_busy: the terminal's input port is full", + ), + InputRefused::TooLarge => std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "terminal input is larger than one host frame", + ), + InputRefused::NoWriter(error) => error, + } + } +} + +struct PortFrame { + kind: MessageKind, + bytes: Vec, +} + +#[derive(Default)] +struct PortState { + frames: VecDeque, + queued_bytes: usize, + writer_running: bool, + closed: bool, +} + +#[derive(Default)] +struct PortShared { + state: Mutex, + changed: Condvar, +} + +/// One terminal's input port (see the module docs). Closed when the terminal drops. +#[derive(Default)] +pub(crate) struct InputPort { + shared: Arc, +} + +impl Drop for InputPort { + fn drop(&mut self) { + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).closed = true; + self.shared.changed.notify_all(); + } +} + +impl InputPort { + /// Takes `bytes` for `terminal`'s current host connection and returns at once; the terminal's writer thread + /// writes them after every earlier write of the port. + pub(crate) fn enqueue( + &self, + terminal: &Arc, + kind: MessageKind, + bytes: &[u8], + ) -> Result<(), InputRefused> { + if bytes.len() > MAX_FRAME_PAYLOAD { + return Err(InputRefused::TooLarge); + } + let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); + let total = state.queued_bytes.checked_add(bytes.len()); + if total.is_none_or(|total| total > INPUT_PORT_BYTE_CAPACITY) { + return Err(InputRefused::Busy); + } + if !state.writer_running { + let shared = Arc::clone(&self.shared); + let terminal = Arc::downgrade(terminal); + std::thread::Builder::new() + .name("terminal-input-port".into()) + .stack_size(INPUT_PORT_STACK_BYTES) + .spawn(move || run_writer(&shared, &terminal)) + .map_err(InputRefused::NoWriter)?; + state.writer_running = true; + } + state.queued_bytes = total.unwrap_or(state.queued_bytes); + state.frames.push_back(PortFrame { kind, bytes: bytes.to_vec() }); + drop(state); + self.shared.changed.notify_one(); + Ok(()) + } +} + +/// The writer: the next frame in order, or `None` when the port closed or stayed idle for +/// [`INPUT_PORT_IDLE_EXIT`] (it then marks itself stopped, under the lock, so the next input starts a new one). +fn next_frame(shared: &PortShared) -> Option { + let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); + let idle_until = Instant::now() + INPUT_PORT_IDLE_EXIT; + loop { + if let Some(frame) = state.frames.pop_front() { + state.queued_bytes = state.queued_bytes.saturating_sub(frame.bytes.len()); + return Some(frame); + } + let now = Instant::now(); + if state.closed || now >= idle_until { + state.writer_running = false; + return None; + } + state = shared + .changed + .wait_timeout(state, idle_until.saturating_duration_since(now)) + .unwrap_or_else(PoisonError::into_inner) + .0; + } +} + +fn run_writer(shared: &PortShared, terminal: &Weak) { + while let Some(frame) = next_frame(shared) { + // The terminal is gone: its port closed, and nothing is left to write to. + let Some(runtime) = terminal.upgrade() else { return }; + if let Err(error) = write_with_reconnect(&runtime, frame.kind, &frame.bytes) { + eprintln!( + "cmux-tui: terminal input not written ({} bytes, already accepted): {error}", + frame.bytes.len() + ); + } + } +} + +/// One frame on the terminal's current host connection, across a lost connection: the rule of +/// `Surface::send_hosted_input`, on the port's thread, reading the published route instead of the runtime lock. +fn write_with_reconnect( + runtime: &PtyTerminalRuntime, + kind: MessageKind, + bytes: &[u8], +) -> std::io::Result<()> { + let deadline = Instant::now() + HOST_INPUT_RECONNECT_WAIT; + let mut first_error = None; + loop { + // Read the revision before the send: an install after it wakes the wait below, so no reconnect is missed. + let observed = runtime.stream_progress.revision(); + let route = runtime.input_route.load_full(); + let sent = match route.as_deref() { + Some(InputRoute::Hosted(route)) => route.send(kind, bytes), + // Keep-on-exit: the final screen drops input; input that already failed keeps its error. + Some(InputRoute::Exited) => return first_error.map_or(Ok(()), Err), + None => { + return Err(first_error.unwrap_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "terminal has no host connection for its input port", + ) + })); + } + }; + let error = match sent { + Ok(()) => return Ok(()), + Err(error) if super::input::host_connection_lost(&error) => error, + Err(error) => return Err(error), + }; + let error = first_error.take().unwrap_or(error); + let state = TerminalHostConnectionState::from_u8( + runtime.host_connection_state.load(Ordering::Acquire), + ); + if matches!( + state, + TerminalHostConnectionState::Failed | TerminalHostConnectionState::Exited + ) || runtime.reader_completion.is_complete() + || runtime.owner_detaching.load(Ordering::Acquire) + || Instant::now() >= deadline + || runtime.stream_progress.wait_for_change(observed, deadline).is_none() + { + return Err(error); + } + first_error = Some(error); + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs b/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs index 4d094c5c4b4a..ffad73023b56 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/spawn.rs @@ -227,6 +227,10 @@ impl Surface { "pty.mouse_encoders", Box::new(mouse_encoders), ), + #[cfg(any(unix, windows))] + input_route: arc_swap::ArcSwapOption::empty(), + #[cfg(any(unix, windows))] + input_port: Default::default(), runtime: RankedMutex::new( LockRank::Runtime, "pty.runtime", diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index 7747ac0a2a0b..99c787177673 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -51,6 +51,30 @@ impl Drop for ReaderCompletionGuard { } } +impl PtyTerminalRuntime { + /// The input route of `runtime`, for the input port (R-DAEMON-STATE stage S1 step 6). + #[cfg(any(unix, windows))] + pub(super) fn input_route_of( + runtime: &PtyRuntime, + ) -> Option> { + match runtime { + PtyRuntime::Hosted(host) => { + Some(Arc::new(super::input_port::InputRoute::Hosted(host.input_route()))) + } + PtyRuntime::ExitedHosted => Some(Arc::new(super::input_port::InputRoute::Exited)), + PtyRuntime::Local { .. } => None, + } + } + + /// Replaces the terminal's runtime and republishes its input route, under the runtime lock the caller holds. + /// Every replacement goes through here, so the port always writes to the connection the runtime has. + #[cfg(any(unix, windows))] + pub(super) fn replace_runtime(&self, runtime: &mut PtyRuntime, next: PtyRuntime) { + *runtime = next; + self.input_route.store(Self::input_route_of(runtime)); + } +} + impl PtyTerminalRuntime { /// Feed raw child output to the generic terminal metadata parser. The /// parser has no knowledge of agents or plugins and keeps only bounded @@ -310,6 +334,14 @@ pub struct PtyTerminalRuntime { /// The view a key may skip its sizing note for, published by the terminal's sizing entry (R-DAEMON-STATE stage /// S1, `mux::sizing_fast_path`); shared by every view of the terminal. pub(super) sizing_fast_path: crate::mux::sizing_fast_path::SizingFastPathSlot, + /// The terminal's input port: on the input lane, interactive input to a hosted terminal goes through it + /// (R-DAEMON-STATE stage S1 step 6, `surface::input_port`). + #[cfg(any(unix, windows))] + pub(super) input_port: super::input_port::InputPort, + /// Where the port writes: the current host connection, the exited final screen, or `None` (not hosted). + /// Published by [`Self::replace_runtime`] and the constructors, so the port never takes the runtime lock. + #[cfg(any(unix, windows))] + pub(super) input_route: arc_swap::ArcSwapOption, #[cfg(test)] pub(super) frame_producer_before_upgrade: FrameProducerTestHook, } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs b/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs index e101986e8299..f2d8bfa18666 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/test_spawn.rs @@ -183,6 +183,10 @@ impl Surface { "pty.mouse_encoders", Box::new(mouse_encoders), ), + #[cfg(any(unix, windows))] + input_route: arc_swap::ArcSwapOption::empty(), + #[cfg(any(unix, windows))] + input_port: Default::default(), runtime: RankedMutex::new( LockRank::Runtime, "pty.runtime", diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index dcb053ddd501..9db18ccf775e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -643,6 +643,7 @@ mod unix { #[cfg(any(unix, windows))] pub use shared::attachment::HostAttachment; +pub(crate) use shared::attachment::HostInputRoute; #[cfg(any(unix, windows))] pub(crate) use shared::attachment::launch::adopt_terminal_host_with_kitty_limits; #[cfg(any(unix, windows))] diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs index eb862321c10c..01f1405713cc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs @@ -143,6 +143,22 @@ impl std::fmt::Debug for HostAttachment { } } +/// Where a terminal's interactive input goes without the terminal's runtime lock (R-DAEMON-STATE stage S1 step 6): +/// the writer of one host connection, shared with its attachment. The terminal publishes the route of its current +/// connection whenever the connection changes (`surface::input_port`). +#[derive(Clone)] +pub(crate) struct HostInputRoute { + writer: Arc>, + protocol_version: u16, +} + +impl HostInputRoute { + /// One frame on the route's connection, as `HostAttachment::send` writes it. + pub(crate) fn send(&self, kind: MessageKind, payload: &[u8]) -> std::io::Result<()> { + send_host_frame(&self.writer, self.protocol_version, kind, payload) + } +} + /// One frame on a daemon-to-host connection's writer. pub(crate) fn send_host_frame( writer: &Mutex, @@ -176,6 +192,11 @@ impl HostAttachment { send_host_frame(&self.writer, self.protocol_version, kind, payload) } + /// The route the terminal's input port writes this connection's interactive input on. + pub(crate) fn input_route(&self) -> HostInputRoute { + HostInputRoute { writer: self.writer.clone(), protocol_version: self.protocol_version } + } + pub(crate) fn begin_input_confirmed( &self, payload: &[u8], From 4bf5af5c6aea742961dd88e3ab6a53ba86ea9966 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:57:18 +0000 Subject: [PATCH 25/42] test(cmux-tui-core): after a respawn, keys skip the sizing lock again; the lock tests report a viewport (#34261 review red) Review comment 6619473751 (FIX at 846d412ccf7): 1. A respawn under the same runtime id keeps the sizing entry, which keeps the dead runtime's fast-path slot: the new runtime's slot is never published, so every later key takes Mux::client_sizing twice. The new test respawns the typing terminal, sends one key, then holds the sizing lock: the next keys must not wait. Red on this commit. 2. The lock tests' view never reported a size, so the engine held the grid and the skip went only through that branch. The typing view now reports 100x30 as the app's view does, so its keys skip through the settled-grid check. --- .../src/server/tests/rds_input_lane.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index 2ba39f279a9a..0cb66e889134 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -67,6 +67,9 @@ impl TypingConnection { let (writer, outbound) = captured_writer(); let client = mux.control_clients.register(ClientTransport::Unix, writer.clone()); attach_test_view(&mux, client, terminal.id, &writer); + // The view reports its size, as the app's view does: its grid is then decided and applied, so a key's sizing + // note is skipped through the settled-grid check, not only while no view counts. + mux.resize_surface_for_client(terminal.id, client, 100, 30).unwrap(); let scheduler = Arc::new(ConnectionSurfaceScheduler::new(mux.surface_operation_admission.clone())); Self { mux, client, writer, outbound, scheduler, surface: terminal.id, input } @@ -356,6 +359,39 @@ fn a_key_waits_for_neither_the_sizing_nor_the_activity_lock() { ]); } +/// A respawn under the same placement and runtime id builds a new terminal runtime: its keys must get the sizing +/// fast path back (the sizing entry outlives the respawn, and must follow it to the new runtime), so after one key +/// that takes the slow path, keys again wait for no sizing lock. +#[test] +fn after_a_respawn_keys_wait_for_no_sizing_lock_again() { + let connection = warmed(TypingConnection::open()); + let removed = connection + .mux + .remove_surface_runtime_for_test(connection.surface) + .expect("the placement had a runtime"); + let respawned = + crate::Surface::spawn_for_test(connection.surface, SurfaceOptions::default(), Weak::new()) + .unwrap(); + let (sent, input) = mpsc::channel(); + respawned.replace_input_writer_for_test(Box::new(TimedInput(sent))); + connection.mux.insert_surface_runtime_for_test(respawned); + let connection = TypingConnection { input, ..connection }; + // One key on the slow path: it finds the sizing entry and, with the fix, points it at the new runtime. + connection.submit("after-respawn", b'r'); + assert!(connection.arrival(b'r').is_some(), "the key after the respawn never arrived"); + connection.settle(&["after-respawn"]); + let waited = key_wait_while_held(&connection, 0, hold_client_sizing); + eprintln!( + "SHARED-LOCKS after a respawn, key wait while client sizing is held {HOLD:?}: {waited:?}" + ); + drop(removed); + connection.close(); + assert!( + waited < BUDGET, + "after a respawn the keys waited {waited:?} for the sizing lock held {HOLD:?}" + ); +} + /// The skipped note must never keep the grid from a view: under the default policy (the newest view sets the grid), /// another view's focus click takes the grid, and the next key from the first view takes it back, on both arms. #[test] From 5fe770df4ff8392c3baab47c7b231c946cb39425 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:04:17 +0000 Subject: [PATCH 26/42] fix(cmux-tui-core): a sizing entry follows its terminal across a respawn; the input time never goes back (#34261 review) Review comment 6619473751 (FIX at 846d412ccf7): 1. A respawn under the same runtime id keeps the sizing entry but builds a new terminal runtime with a new fast-path slot. TerminalSizingEntry::follow_runtime_of re-points the entry at the slot of the runtime its keys reach, under the sizing lock, and clears the old slot: on the key's slow path (note_terminal_activity_of has the key's surface) and on every terminal_sizing_entry call (attach, claim, size report). The red test after_a_respawn_keys_wait_for_no_sizing_lock_again (4bf5af5c6ae) pins it. 3. The entry takes or follows a slot only from a surface whose runtime id is the entry's, so two entries never share one slot. 4. The lock-free input time is fetch_max, so a key preempted after its clock read never moves it back. 7. The poison comment says what happens: a later holder that recovers the poison republishes what it touches. Lock order is unchanged: terminal_sizing_entry now looks the surface up on each call (it did at creation, and apply_terminal_grid does), under the sizing lock, the established sizing-then-session order. --- .../cmux-tui-core/src/mux/sizing_fast_path.rs | 24 ++++++++++++++++++- .../cmux-tui-core/src/mux/terminal_sizing.rs | 14 ++++++++--- .../cmux-tui-core/src/server/activity.rs | 5 ++-- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs b/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs index 080c003347fe..ca64d7527982 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs @@ -132,6 +132,27 @@ impl SizingEntries { } impl TerminalSizingEntry { + /// Points the entry at `surface`'s terminal runtime when that runtime is the entry's: a respawn under the same + /// runtime id builds a new runtime with a new slot, and the entry outlives it. The old slot is cleared; the + /// sizing guard publishes into the new one (the caller reached the entry through `SizingEntries`, which marked + /// it). A surface of another runtime changes nothing. + pub(in crate::mux) fn follow_runtime_of( + &mut self, + runtime: SurfaceId, + surface: &crate::Surface, + ) { + if surface.terminal_runtime_id() != Some(runtime) { + return; + } + let Some(slot) = surface.sizing_fast_path_slot() else { return }; + if self.fast_path_slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &slot)) { + return; + } + if let Some(old) = self.fast_path_slot.replace(slot) { + old.store(None); + } + } + /// The view a key may skip its note for: the sole newest member, a client's own view, while nothing waits for /// the next apply. Settled: no grid hold, no handheld that reported a size (a handheld caps or holds the grid), /// and, unless the engine holds the grid (no view counts, so an apply does nothing), the grid it applied is its @@ -229,7 +250,8 @@ impl Drop for SizingGuard<'_> { fn drop(&mut self) { if self.guard.terminal_sizing.publishing { // A panic that began while this guard held the lock cut its change short: the marked entries publish - // nothing (their keys take the slow path, which meets the poisoned lock as the old path does). + // nothing, so their keys take the slow path. A later holder that recovers the poison republishes what it + // touches (the settled and member checks still apply to what it sees). let cut_short = std::thread::panicking() && !self.panicking_at_lock; self.guard.terminal_sizing.publish_marked(cut_short); } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs index 2f317d4cb082..a10dd769839b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs @@ -91,6 +91,8 @@ impl Mux { if entry.members.get(&id).is_none_or(|member| member.client != client) { return None; } + // The key's terminal may be a respawn of the entry's runtime: its fast path follows it. + entry.follow_runtime_of(runtime, surface); let changed = entry.engine.note_activity(&id); sizing.note_size_state(runtime, changed); Some(changed) @@ -533,11 +535,14 @@ impl Mux { runtime: SurfaceId, placement: SurfaceId, ) -> &'a mut TerminalSizingEntry { + let surface = self.surface(placement).or_else(|| self.surface(runtime)); if !sizing.terminal_sizing.contains_key(&runtime) { - let surface = self.surface(placement).or_else(|| self.surface(runtime)); let (cols, rows) = surface.as_ref().map_or((80, 24), |surface| surface.size()); - let fast_path_slot = - surface.as_ref().and_then(|surface| surface.sizing_fast_path_slot()); + // Only a surface of this runtime: a placement that a respawn moved to a new runtime id keeps its slot. + let fast_path_slot = surface + .as_ref() + .filter(|surface| surface.terminal_runtime_id() == Some(runtime)) + .and_then(|surface| surface.sizing_fast_path_slot()); let policy = self.resolved_size_policy(sizing, runtime, placement); sizing.terminal_sizing.insert( runtime, @@ -553,6 +558,9 @@ impl Mux { ); } let entry = sizing.terminal_sizing.get_mut(&runtime).expect("inserted above"); + if let Some(surface) = &surface { + entry.follow_runtime_of(runtime, surface); + } entry.placements.insert(placement); entry } diff --git a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs index 7a8d44cb04bd..0cde28afba23 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/activity.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/activity.rs @@ -92,8 +92,9 @@ impl ActivityStream { self.update(|state| state.user_input_ms = Some(now_ms())); return; } - // SeqCst: either this key sees the subscriber, or the subscriber's first snapshot sees this time. - self.shared.user_input_ms.store(now_ms(), Ordering::SeqCst); + // SeqCst: either this key sees the subscriber, or the subscriber's first snapshot sees this time. A key + // preempted after its clock read never moves the time back. + self.shared.user_input_ms.fetch_max(now_ms(), Ordering::SeqCst); if self.shared.subscribed.load(Ordering::SeqCst) && !self.shared.wake_pending.swap(true, Ordering::SeqCst) { From 89414e0419e5f6892d3dd6088f169fceded77924 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:08:19 +0000 Subject: [PATCH 27/42] fix(cmux-tui-core): name the input port module without super:: where surface::* is in scope (clippy unused_qualifications) --- cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/surface/input.rs | 4 ++-- .../cmux-tui-core/src/surface/terminal_runtime.rs | 12 +++++------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs b/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs index 08bef432a7e0..fee40fee2d66 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/adopt.rs @@ -230,7 +230,7 @@ impl Surface { Box::new(mouse_encoders), ), input_route: arc_swap::ArcSwapOption::new(Some(Arc::new( - super::input_port::InputRoute::Exited, + input_port::InputRoute::Exited, ))), input_port: Default::default(), runtime: RankedMutex::new( diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index 00de6f4ccb6d..88aa03ce584f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -75,8 +75,8 @@ impl Surface { } match pty.input_route.load().as_deref() { None => None, - Some(super::input_port::InputRoute::Exited) => Some(Ok(())), - Some(super::input_port::InputRoute::Hosted(_)) => { + Some(input_port::InputRoute::Exited) => Some(Ok(())), + Some(input_port::InputRoute::Hosted(_)) => { Some(pty.input_port.enqueue(&pty.terminal, kind, bytes).map_err(Into::into)) } } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index 99c787177673..a821829deb01 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -54,14 +54,12 @@ impl Drop for ReaderCompletionGuard { impl PtyTerminalRuntime { /// The input route of `runtime`, for the input port (R-DAEMON-STATE stage S1 step 6). #[cfg(any(unix, windows))] - pub(super) fn input_route_of( - runtime: &PtyRuntime, - ) -> Option> { + pub(super) fn input_route_of(runtime: &PtyRuntime) -> Option> { match runtime { PtyRuntime::Hosted(host) => { - Some(Arc::new(super::input_port::InputRoute::Hosted(host.input_route()))) + Some(Arc::new(input_port::InputRoute::Hosted(host.input_route()))) } - PtyRuntime::ExitedHosted => Some(Arc::new(super::input_port::InputRoute::Exited)), + PtyRuntime::ExitedHosted => Some(Arc::new(input_port::InputRoute::Exited)), PtyRuntime::Local { .. } => None, } } @@ -337,11 +335,11 @@ pub struct PtyTerminalRuntime { /// The terminal's input port: on the input lane, interactive input to a hosted terminal goes through it /// (R-DAEMON-STATE stage S1 step 6, `surface::input_port`). #[cfg(any(unix, windows))] - pub(super) input_port: super::input_port::InputPort, + pub(super) input_port: input_port::InputPort, /// Where the port writes: the current host connection, the exited final screen, or `None` (not hosted). /// Published by [`Self::replace_runtime`] and the constructors, so the port never takes the runtime lock. #[cfg(any(unix, windows))] - pub(super) input_route: arc_swap::ArcSwapOption, + pub(super) input_route: arc_swap::ArcSwapOption, #[cfg(test)] pub(super) frame_producer_before_upgrade: FrameProducerTestHook, } From 8ccc6f0f0db5f00ea69b81f9e901e0cfd90c05d3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:40:51 +0000 Subject: [PATCH 28/42] test(cmux-tui-core): a settled size-state read takes the session lock once with the switch off (#34261 review 2 red) The sizing entry looks its surface up on every access since 5fe770df4ff, so with the switch off a size-state read takes the session lock twice (once under the sizing lock): expected red at this commit (lane off: 2, not 1). --- .../src/server/tests/rds_input_lane.rs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index 0cb66e889134..bd45a15de40f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -392,6 +392,35 @@ fn after_a_respawn_keys_wait_for_no_sizing_lock_again() { ); } +/// #34261 review 2: a settled terminal's sizing entry looks its surface up only when it makes the entry, so a size +/// state read takes the session lock once with the switch off (the caller's own lookup), never again under the +/// sizing lock, and none with the switch on. Looked up on every access, it took the session lock under the sizing +/// lock in every attach, size report, claim and size-state read, so with the switch off (the default) a key's sizing +/// note waited for automation's session-lock holds behind them. +#[test] +fn a_settled_size_state_read_takes_the_session_lock_once_with_the_switch_off() { + let mut counts = Vec::new(); + for (lane, expected) in [(false, 1), (true, 0)] { + let connection = TypingConnection::open_with(lane); + // The view's attach and its 100x30 report made the entry; this read settles nothing new. + assert!(connection.mux.terminal_size_state(connection.surface).is_some()); + let before = crate::mux::state_locks_taken_on_this_thread(); + assert!(connection.mux.terminal_size_state(connection.surface).is_some()); + let taken = crate::mux::state_locks_taken_on_this_thread() - before; + connection.close(); + counts.push((lane, taken, expected)); + } + eprintln!( + "SHARED-LOCKS session locks per settled size-state read (lane, taken, expected): {counts:?}" + ); + for (lane, taken, expected) in counts { + assert_eq!( + taken, expected, + "lane {lane}: one settled size-state read took the session lock {taken} times, not {expected}" + ); + } +} + /// The skipped note must never keep the grid from a view: under the default policy (the newest view sets the grid), /// another view's focus click takes the grid, and the next key from the first view takes it back, on both arms. #[test] From 8701dbd8433e665830be66a6606cd9b820e78c1b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:43:13 +0000 Subject: [PATCH 29/42] fix(cmux-tui-core): a sizing entry looks its surface up only when it is made; the respawn follow runs only on the lane (#34261 review 2) 5fe770df4ff looked the surface up on every sizing-entry access, so with the switch off (the default) every attach, size report, claim and size-state read took the session lock under the sizing lock. The surface is looked up only to make the entry again; a respawn's new runtime is followed by its first key's sizing note, which holds the surface, and only while the switch is on. An unchanged fast path is not stored again. The respawn test pins the old slot's clear. --- .../cmux-tui-core/src/mux/sizing_fast_path.rs | 15 ++++++++++++--- .../cmux-tui-core/src/mux/terminal_sizing.rs | 16 ++++++++++------ .../src/server/tests/rds_input_lane.rs | 5 +++++ 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs b/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs index ca64d7527982..67bafc1818ab 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/sizing_fast_path.rs @@ -86,6 +86,11 @@ impl SizingEntries { removed } + /// Whether the entries publish their fast paths (the mux's stage S1 switch). + pub(in crate::mux) fn publishing(&self) -> bool { + self.publishing + } + pub(in crate::mux) fn contains_key(&self, runtime: &SurfaceId) -> bool { self.map.contains_key(runtime) } @@ -113,8 +118,11 @@ impl SizingEntries { for runtime in touched { let Some(entry) = self.map.get(&runtime) else { continue }; let Some(slot) = entry.fast_path_slot.as_ref() else { continue }; - let published = if cut_short { None } else { entry.fast_path().map(Arc::new) }; - slot.store(published); + let published = if cut_short { None } else { entry.fast_path() }; + // An unchanged value is not stored again: a read that changed nothing allocates nothing. + if slot.load().as_deref().copied() != published { + slot.store(published.map(Arc::new)); + } } } @@ -135,7 +143,8 @@ impl TerminalSizingEntry { /// Points the entry at `surface`'s terminal runtime when that runtime is the entry's: a respawn under the same /// runtime id builds a new runtime with a new slot, and the entry outlives it. The old slot is cleared; the /// sizing guard publishes into the new one (the caller reached the entry through `SizingEntries`, which marked - /// it). A surface of another runtime changes nothing. + /// it). A surface of another runtime changes nothing. Called by a key's sizing note, which holds the surface, so + /// it costs no lookup. pub(in crate::mux) fn follow_runtime_of( &mut self, runtime: SurfaceId, diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs index a10dd769839b..2f87d637f956 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/terminal_sizing.rs @@ -87,12 +87,16 @@ impl Mux { // a key can change, so a key does not re-expand it: that asks for the terminal's workspace default, which // reads the session lock (TYPING FIX 2). self.mutate_terminal_sizing_keeping_participants(runtime, |_, sizing| { + let publishing = sizing.terminal_sizing.publishing(); let entry = sizing.terminal_sizing.get_mut(&runtime)?; if entry.members.get(&id).is_none_or(|member| member.client != client) { return None; } - // The key's terminal may be a respawn of the entry's runtime: its fast path follows it. - entry.follow_runtime_of(runtime, surface); + // The key's terminal may be a respawn of the entry's runtime: its fast path follows it (stage S1 only; + // with the switch off nothing is published). + if publishing { + entry.follow_runtime_of(runtime, surface); + } let changed = entry.engine.note_activity(&id); sizing.note_size_state(runtime, changed); Some(changed) @@ -535,8 +539,11 @@ impl Mux { runtime: SurfaceId, placement: SurfaceId, ) -> &'a mut TerminalSizingEntry { - let surface = self.surface(placement).or_else(|| self.surface(runtime)); + // The surface is looked up only to make the entry: with the switch off the lookup takes the session lock, + // under the sizing lock (#34261 review 2). A respawn's new runtime is followed by its first key's sizing + // note (`note_terminal_activity_of`), which holds the surface already. if !sizing.terminal_sizing.contains_key(&runtime) { + let surface = self.surface(placement).or_else(|| self.surface(runtime)); let (cols, rows) = surface.as_ref().map_or((80, 24), |surface| surface.size()); // Only a surface of this runtime: a placement that a respawn moved to a new runtime id keeps its slot. let fast_path_slot = surface @@ -558,9 +565,6 @@ impl Mux { ); } let entry = sizing.terminal_sizing.get_mut(&runtime).expect("inserted above"); - if let Some(surface) = &surface { - entry.follow_runtime_of(runtime, surface); - } entry.placements.insert(placement); entry } diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index bd45a15de40f..b52020b5df90 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -380,6 +380,11 @@ fn after_a_respawn_keys_wait_for_no_sizing_lock_again() { connection.submit("after-respawn", b'r'); assert!(connection.arrival(b'r').is_some(), "the key after the respawn never arrived"); connection.settle(&["after-respawn"]); + // The old runtime's slot is cleared, so nothing publishes into a runtime that no view types into. + assert!( + !removed.skips_sizing_note(connection.client), + "the replaced runtime still publishes the sizing fast path" + ); let waited = key_wait_while_held(&connection, 0, hold_client_sizing); eprintln!( "SHARED-LOCKS after a respawn, key wait while client sizing is held {HOLD:?}: {waited:?}" From 417fab72381d5dbc0b96a6473166d27aa03195ba Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:49:16 +0000 Subject: [PATCH 30/42] fix(cmux-tui-core): the input port's order test can fail; the port copies outside its lock, counts each frame, recovers from a writer panic and refuses a failed link (#34544 review) - The test compared two Options (None < Some), so a key that passed the paste still passed; both positions are required now. A new test sends 26 keys on the lane and checks the host reads each once, in order. - enqueue copies the bytes before the port lock, and each queued frame counts 64 bytes of bookkeeping against the 32 MiB capacity, so many tiny frames cannot grow the queue without bound. - A writer that unwinds drops the frames after the one it lost (logged) and marks itself stopped, so the next input starts a new writer instead of queueing behind a writer that is gone. - A host connection the reader gave up on (Failed) refuses input at once, as the old path's failed write did. - The deadlines use checked_add (no new arithmetic_side_effects hits); the thread name fits Linux's 15 bytes. --- .../crates/cmux-tui-core/src/surface/input.rs | 12 ++++ .../cmux-tui-core/src/surface/input_port.rs | 63 ++++++++++++++++--- .../cmux-tui-core/src/surface/tests/hosted.rs | 31 ++++++++- 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index 88aa03ce584f..385e9b80b29b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -76,6 +76,18 @@ impl Surface { match pty.input_route.load().as_deref() { None => None, Some(input_port::InputRoute::Exited) => Some(Ok(())), + // A connection the reader gave up on takes no input: refused now, as the old path's failed write was, + // so the sender learns it (`cmux send` exits non-zero) instead of an accepted write the port only logs. + Some(input_port::InputRoute::Hosted(_)) + if TerminalHostConnectionState::from_u8( + pty.host_connection_state.load(Ordering::Acquire), + ) == TerminalHostConnectionState::Failed => + { + Some(Err(std::io::Error::new( + std::io::ErrorKind::NotConnected, + "the terminal's host connection failed; input not written", + ))) + } Some(input_port::InputRoute::Hosted(_)) => { Some(pty.input_port.enqueue(&pty.terminal, kind, bytes).map_err(Into::into)) } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index 0523b3d7828a..9163d0cbb27b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -15,7 +15,10 @@ //! time the connection changes (`PtyTerminalRuntime::replace_runtime`), so it never takes the runtime lock either. //! - A lost host connection is retried as `write_bytes` did (cx-6so.49): the writer waits for the reader's next //! stream change and writes to the new host, until `HOST_INPUT_RECONNECT_WAIT`. -//! - A full port refuses at once with a typed error: nothing is dropped silently and nothing waits. +//! - A full port refuses at once with a typed error: nothing is dropped silently and nothing waits. Each queued +//! frame counts its bookkeeping too ([`PORT_FRAME_COST`]), so many tiny frames cannot grow the queue without bound. +//! - A host connection the reader gave up on (`Failed`) takes no input: the write is refused at once, as the old +//! path's failed write was (`Surface::enqueue_on_input_port`). //! - Input the port took and then could not write (the host is gone and no new one came) is lost, as a failed write //! was before; the request had already been answered, so the port logs it. //! - Stage S1 step 6b, before the default flip: every other frame of the connection (viewer size, cell pixels, @@ -34,6 +37,8 @@ use crate::terminal_host_runtime::HostInputRoute; /// The most bytes one terminal's port holds; more is refused (`terminal.input_busy`). Two of the largest frames. const INPUT_PORT_BYTE_CAPACITY: usize = 2 * MAX_FRAME_PAYLOAD; +/// What one queued frame costs beyond its bytes (its header and allocation), counted against the capacity. +const PORT_FRAME_COST: usize = 64; /// A port writer with nothing to write for this long ends; the next input starts a new one. const INPUT_PORT_IDLE_EXIT: Duration = Duration::from_secs(30); /// The writer's stack: it formats and writes one frame at a time. @@ -79,6 +84,13 @@ struct PortFrame { bytes: Vec, } +impl PortFrame { + /// What the frame counts against [`INPUT_PORT_BYTE_CAPACITY`]: its bytes and [`PORT_FRAME_COST`]. + fn cost(&self) -> usize { + self.bytes.len().saturating_add(PORT_FRAME_COST) + } +} + #[derive(Default)] struct PortState { frames: VecDeque, @@ -118,23 +130,29 @@ impl InputPort { if bytes.len() > MAX_FRAME_PAYLOAD { return Err(InputRefused::TooLarge); } + // Copied before the lock, so another write to this terminal never waits for this one's copy. + let frame = PortFrame { kind, bytes: bytes.to_vec() }; let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); - let total = state.queued_bytes.checked_add(bytes.len()); - if total.is_none_or(|total| total > INPUT_PORT_BYTE_CAPACITY) { + let Some(total) = state + .queued_bytes + .checked_add(frame.cost()) + .filter(|total| *total <= INPUT_PORT_BYTE_CAPACITY) + else { return Err(InputRefused::Busy); - } + }; if !state.writer_running { let shared = Arc::clone(&self.shared); let terminal = Arc::downgrade(terminal); std::thread::Builder::new() - .name("terminal-input-port".into()) + // At most 15 bytes: the length Linux keeps. + .name("term-input-port".into()) .stack_size(INPUT_PORT_STACK_BYTES) .spawn(move || run_writer(&shared, &terminal)) .map_err(InputRefused::NoWriter)?; state.writer_running = true; } - state.queued_bytes = total.unwrap_or(state.queued_bytes); - state.frames.push_back(PortFrame { kind, bytes: bytes.to_vec() }); + state.queued_bytes = total; + state.frames.push_back(frame); drop(state); self.shared.changed.notify_one(); Ok(()) @@ -145,10 +163,10 @@ impl InputPort { /// [`INPUT_PORT_IDLE_EXIT`] (it then marks itself stopped, under the lock, so the next input starts a new one). fn next_frame(shared: &PortShared) -> Option { let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); - let idle_until = Instant::now() + INPUT_PORT_IDLE_EXIT; + let idle_until = Instant::now().checked_add(INPUT_PORT_IDLE_EXIT).unwrap_or_else(Instant::now); loop { if let Some(frame) = state.frames.pop_front() { - state.queued_bytes = state.queued_bytes.saturating_sub(frame.bytes.len()); + state.queued_bytes = state.queued_bytes.saturating_sub(frame.cost()); return Some(frame); } let now = Instant::now(); @@ -164,7 +182,31 @@ fn next_frame(shared: &PortShared) -> Option { } } +/// Stops the port cleanly when its writer unwinds: the frame it was writing is lost, so the frames after it are +/// dropped too (a paste without its start, keys without the ones before them) and logged, and the writer is marked +/// stopped, so the next input starts a new one instead of queueing behind a writer that is gone. Only on unwind: a +/// normal exit marks the writer stopped under the lock in [`next_frame`], and a newer writer may run by then. +struct WriterUnwind<'a>(&'a PortShared); + +impl Drop for WriterUnwind<'_> { + fn drop(&mut self) { + if !std::thread::panicking() { + return; + } + let mut state = self.0.state.lock().unwrap_or_else(PoisonError::into_inner); + let dropped = state.frames.len(); + state.frames.clear(); + state.queued_bytes = 0; + state.writer_running = false; + drop(state); + eprintln!( + "cmux-tui: terminal input port writer stopped by a panic; {dropped} queued writes dropped" + ); + } +} + fn run_writer(shared: &PortShared, terminal: &Weak) { + let _unwind = WriterUnwind(shared); while let Some(frame) = next_frame(shared) { // The terminal is gone: its port closed, and nothing is left to write to. let Some(runtime) = terminal.upgrade() else { return }; @@ -184,7 +226,8 @@ fn write_with_reconnect( kind: MessageKind, bytes: &[u8], ) -> std::io::Result<()> { - let deadline = Instant::now() + HOST_INPUT_RECONNECT_WAIT; + let deadline = + Instant::now().checked_add(HOST_INPUT_RECONNECT_WAIT).unwrap_or_else(Instant::now); let mut first_error = None; loop { // Read the revision before the send: an install after it wakes the wait below, so no reconnect is missed. diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index 7dc1f5ccd5e3..43683b9fc7bb 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -186,13 +186,40 @@ fn a_key_does_not_wait_for_another_writers_blocked_host_socket() { 1, "the key reached the host once: {kinds:?}" ); + // Both must be there: the host stops reading at the first key, so a key that passed the paste leaves no paste. + let paste_at = kinds.iter().position(|kind| *kind == MessageKind::Paste); + let key_at = kinds.iter().position(|kind| *kind == MessageKind::Input); assert!( - kinds.iter().position(|kind| *kind == MessageKind::Paste) - < kinds.iter().position(|kind| *kind == MessageKind::Input), + matches!((paste_at, key_at), (Some(paste), Some(key)) if paste < key), "the paste, written first, reached the host before the key: {kinds:?}" ); } +/// Keys to a hosted terminal on the input port reach the host once each, in the order they were typed (the port's +/// one writer thread), and a key the port took returns before the host read anything (#34544 review). +#[cfg(unix)] +#[test] +fn keys_on_the_input_port_reach_the_host_once_each_in_order() { + let (_mux, surface, mut host) = hosted_typing_surface("hosted-input-port-order", true); + let keys: Vec = (b'a'..=b'z').collect(); + for key in &keys { + surface.write_bytes(&[*key]).expect("the port took the key"); + } + let mut written = Vec::new(); + while written.len() < keys.len() { + let frame = crate::terminal_host_protocol::read_frame( + &mut host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) + .expect("the host reads the keys") + .expect("the host connection stays open"); + if frame.kind == MessageKind::Input { + written.extend_from_slice(&frame.payload); + } + } + assert_eq!(written, keys, "the keys reached the host in another order or more than once"); +} + #[cfg(unix)] #[test] fn hosted_stager_exposes_coupled_state_only_after_colors() { From 7c90c2965f6108d9f3f20d7fe73f754c6b9dfd70 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 04:59:36 +0000 Subject: [PATCH 31/42] fix(cmux-tui-core): the input port refuses only when the host reader ended, not while a Failed link still retries (#34544 review 2) A Failed link keeps retrying its host, and a successful retry installs the live replacement before the state leaves Failed; the old path wrote to it, the lane refused. The lane now refuses only when the reader ended (no connection will come back), where the old path's write fails too. The capacity holds two of the largest frames with their per-frame cost; the 26-key test also checks that no key is written twice. --- .../crates/cmux-tui-core/src/surface/input.rs | 13 +++++------- .../cmux-tui-core/src/surface/input_port.rs | 10 ++++++---- .../cmux-tui-core/src/surface/tests/hosted.rs | 20 ++++++++++++++++++- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index 385e9b80b29b..d67dea3264c8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -76,16 +76,13 @@ impl Surface { match pty.input_route.load().as_deref() { None => None, Some(input_port::InputRoute::Exited) => Some(Ok(())), - // A connection the reader gave up on takes no input: refused now, as the old path's failed write was, - // so the sender learns it (`cmux send` exits non-zero) instead of an accepted write the port only logs. - Some(input_port::InputRoute::Hosted(_)) - if TerminalHostConnectionState::from_u8( - pty.host_connection_state.load(Ordering::Acquire), - ) == TerminalHostConnectionState::Failed => - { + // The reader ended: no host connection will come back, so the input is refused now, as the old path's + // write failed then (`cmux send` exits non-zero). `Failed` alone is not that: the reader keeps retrying, + // and it installs a live replacement before the state leaves `Failed` (#34544 review 2). + Some(input_port::InputRoute::Hosted(_)) if pty.reader_completion.is_complete() => { Some(Err(std::io::Error::new( std::io::ErrorKind::NotConnected, - "the terminal's host connection failed; input not written", + "the terminal's host connection ended; input not written", ))) } Some(input_port::InputRoute::Hosted(_)) => { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index 9163d0cbb27b..39fe12d93237 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -17,8 +17,9 @@ //! stream change and writes to the new host, until `HOST_INPUT_RECONNECT_WAIT`. //! - A full port refuses at once with a typed error: nothing is dropped silently and nothing waits. Each queued //! frame counts its bookkeeping too ([`PORT_FRAME_COST`]), so many tiny frames cannot grow the queue without bound. -//! - A host connection the reader gave up on (`Failed`) takes no input: the write is refused at once, as the old -//! path's failed write was (`Surface::enqueue_on_input_port`). +//! - A terminal whose host reader ended (no connection will come back) takes no input: the write is refused at once, +//! as the old path's write failed then (`Surface::enqueue_on_input_port`). A `Failed` connection whose reader still +//! retries is accepted, as in any reconnect wait. //! - Input the port took and then could not write (the host is gone and no new one came) is lost, as a failed write //! was before; the request had already been answered, so the port logs it. //! - Stage S1 step 6b, before the default flip: every other frame of the connection (viewer size, cell pixels, @@ -35,10 +36,11 @@ use super::{PtyTerminalRuntime, TerminalHostConnectionState}; use crate::terminal_host_protocol::{MAX_FRAME_PAYLOAD, MessageKind}; use crate::terminal_host_runtime::HostInputRoute; -/// The most bytes one terminal's port holds; more is refused (`terminal.input_busy`). Two of the largest frames. -const INPUT_PORT_BYTE_CAPACITY: usize = 2 * MAX_FRAME_PAYLOAD; /// What one queued frame costs beyond its bytes (its header and allocation), counted against the capacity. const PORT_FRAME_COST: usize = 64; +/// The most one terminal's port holds; more is refused (`terminal.input_busy`). Two of the largest frames, with +/// their cost. +const INPUT_PORT_BYTE_CAPACITY: usize = 2 * (MAX_FRAME_PAYLOAD + PORT_FRAME_COST); /// A port writer with nothing to write for this long ends; the next input starts a new one. const INPUT_PORT_IDLE_EXIT: Duration = Duration::from_secs(30); /// The writer's stack: it formats and writes one frame at a time. diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index 43683b9fc7bb..21e7a7551a0c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -181,10 +181,12 @@ fn a_key_does_not_wait_for_another_writers_blocked_host_socket() { took < Duration::from_millis(100), "the key waited {took:?} for another writer's blocked host socket" ); + // The host stops reading at the first key frame, so this proves the key arrived (after the paste, below), not + // that it arrived only once: `keys_on_the_input_port_reach_the_host_once_each_in_order` checks that. assert_eq!( kinds.iter().filter(|kind| **kind == MessageKind::Input).count(), 1, - "the key reached the host once: {kinds:?}" + "the key reached the host: {kinds:?}" ); // Both must be there: the host stops reading at the first key, so a key that passed the paste leaves no paste. let paste_at = kinds.iter().position(|kind| *kind == MessageKind::Paste); @@ -218,6 +220,22 @@ fn keys_on_the_input_port_reach_the_host_once_each_in_order() { } } assert_eq!(written, keys, "the keys reached the host in another order or more than once"); + // Nothing more: no key is written twice. + host.set_read_timeout(Some(Duration::from_millis(150))).unwrap(); + loop { + match crate::terminal_host_protocol::read_frame( + &mut host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) { + Ok(Some(frame)) => assert_ne!( + frame.kind, + MessageKind::Input, + "a key reached the host twice: {:?}", + frame.payload + ), + Ok(None) | Err(_) => break, + } + } } #[cfg(unix)] From 2dd59d8033cd9fbcea749331ce5bb32f3a411608 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:35:17 +0000 Subject: [PATCH 32/42] fix(cmux-tui-core): a key to a failed host link is refused at once on the input port, as on the old path (#34544 review 3) When the reader moves a link to Failed it shuts the connection down (and keeps retrying); the old path's write then failed and `cmux send` exited non-zero, while the port took the key and only logged its failure. The reader now publishes InputRoute::Failed under the runtime lock when it shuts the connection down; the port refuses it with NotConnected, and the replacement's route (replace_runtime) is accepted as soon as it is installed. Also the 26-key test's tail loop is a while-let (clippy::while_let_loop). Test: a_key_to_a_failed_link_is_refused_until_a_live_route_is_published. --- .../crates/cmux-tui-core/src/surface/input.rs | 4 ++ .../cmux-tui-core/src/surface/input_port.rs | 18 +++++-- .../src/surface/reconnect_backoff.rs | 9 ++-- .../src/surface/terminal_runtime.rs | 8 +++ .../cmux-tui-core/src/surface/tests/hosted.rs | 53 ++++++++++++++----- 5 files changed, 73 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index d67dea3264c8..0481cfa86fda 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -76,6 +76,10 @@ impl Surface { match pty.input_route.load().as_deref() { None => None, Some(input_port::InputRoute::Exited) => Some(Ok(())), + Some(input_port::InputRoute::Failed) => Some(Err(std::io::Error::new( + std::io::ErrorKind::NotConnected, + "the terminal's host connection failed; input not written", + ))), // The reader ended: no host connection will come back, so the input is refused now, as the old path's // write failed then (`cmux send` exits non-zero). `Failed` alone is not that: the reader keeps retrying, // and it installs a live replacement before the state leaves `Failed` (#34544 review 2). diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index 39fe12d93237..6f56cfe7650a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -17,9 +17,10 @@ //! stream change and writes to the new host, until `HOST_INPUT_RECONNECT_WAIT`. //! - A full port refuses at once with a typed error: nothing is dropped silently and nothing waits. Each queued //! frame counts its bookkeeping too ([`PORT_FRAME_COST`]), so many tiny frames cannot grow the queue without bound. -//! - A terminal whose host reader ended (no connection will come back) takes no input: the write is refused at once, -//! as the old path's write failed then (`Surface::enqueue_on_input_port`). A `Failed` connection whose reader still -//! retries is accepted, as in any reconnect wait. +//! - A connection the reader shut down (`Failed`, it keeps retrying: [`InputRoute::Failed`]) or a reader that ended +//! takes no input: the write is refused at once, as the old path's write failed then +//! (`Surface::enqueue_on_input_port`). A replacement's route is accepted as soon as it is installed. While a lost +//! connection is still `Reconnecting`, input is accepted and the writer waits for the reconnect. //! - Input the port took and then could not write (the host is gone and no new one came) is lost, as a failed write //! was before; the request had already been answered, so the port logs it. //! - Stage S1 step 6b, before the default flip: every other frame of the connection (viewer size, cell pixels, @@ -52,6 +53,9 @@ pub(crate) enum InputRoute { Hosted(HostInputRoute), /// The host exited and the terminal keeps its final screen: input is dropped, as `write_bytes` drops it. Exited, + /// The reader shut the host connection down and keeps retrying (`Failed`): input is refused until a + /// replacement is installed, as the old path's write to the shut connection failed. + Failed, } /// Why the port did not take a write. @@ -239,6 +243,14 @@ fn write_with_reconnect( Some(InputRoute::Hosted(route)) => route.send(kind, bytes), // Keep-on-exit: the final screen drops input; input that already failed keeps its error. Some(InputRoute::Exited) => return first_error.map_or(Ok(()), Err), + Some(InputRoute::Failed) => { + return Err(first_error.unwrap_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "the terminal's host connection failed; input not written", + ) + })); + } None => { return Err(first_error.unwrap_or_else(|| { std::io::Error::new( diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs b/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs index 57d917f72d51..10c0577a4dbc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs @@ -66,11 +66,14 @@ impl TerminalHostReconnectBackoff { .host_connection_state .swap(TerminalHostConnectionState::Failed as u8, Ordering::AcqRel); if was != TerminalHostConnectionState::Failed as u8 { - if let PtyRuntime::Hosted(host) = - &*pty.runtime.lock().unwrap_or_else(std::sync::PoisonError::into_inner) - { + let runtime = + pty.runtime.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let PtyRuntime::Hosted(host) = &*runtime { host.disconnect(); + // The input port refuses until a replacement is installed, as a write here fails now. + pty.input_route_failed(&runtime); } + drop(runtime); pty.stream_progress.notify(); publish_host_link_state(pty); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index a821829deb01..2d6cccf5f6f6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -71,6 +71,14 @@ impl PtyTerminalRuntime { *runtime = next; self.input_route.store(Self::input_route_of(runtime)); } + + /// The reader shut the current host connection down and keeps retrying (`Failed`): input is refused until a + /// replacement connection is installed ([`Self::replace_runtime`] publishes its route), as the old path's write + /// to the shut connection failed. Takes the held runtime so it is ordered with every replacement. + #[cfg(any(unix, windows))] + pub(super) fn input_route_failed(&self, _runtime: &PtyRuntime) { + self.input_route.store(Some(Arc::new(input_port::InputRoute::Failed))); + } } impl PtyTerminalRuntime { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index 21e7a7551a0c..0d95f635b3d9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -197,6 +197,35 @@ fn a_key_does_not_wait_for_another_writers_blocked_host_socket() { ); } +/// A key to a terminal whose host connection the reader shut down (`Failed`, still retrying) is refused at once, as +/// the old path's write to the shut connection failed (`cmux send` exits non-zero); once a replacement's route is +/// published, keys go through again (#34544 review 3). +#[cfg(unix)] +#[test] +fn a_key_to_a_failed_link_is_refused_until_a_live_route_is_published() { + let (_mux, surface, mut host) = hosted_typing_surface("hosted-input-port-failed", true); + let pty = surface.as_pty().expect("a terminal"); + pty.input_route.store(Some(Arc::new(input_port::InputRoute::Failed))); + let refused = surface.write_bytes(b"x").expect_err("a key to a failed link was accepted"); + assert_eq!(refused.kind(), std::io::ErrorKind::NotConnected, "{refused}"); + + let live = PtyTerminalRuntime::input_route_of(&pty.runtime.lock().unwrap()); + pty.input_route.store(live); + surface.write_bytes(b"y").expect("the key after the replacement was refused"); + let frame = loop { + let frame = crate::terminal_host_protocol::read_frame( + &mut host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) + .expect("the host reads the key") + .expect("the host connection stays open"); + if frame.kind == MessageKind::Input { + break frame; + } + }; + assert_eq!(frame.payload, b"y", "the refused key must not reach the host"); +} + /// Keys to a hosted terminal on the input port reach the host once each, in the order they were typed (the port's /// one writer thread), and a key the port took returns before the host read anything (#34544 review). #[cfg(unix)] @@ -222,19 +251,17 @@ fn keys_on_the_input_port_reach_the_host_once_each_in_order() { assert_eq!(written, keys, "the keys reached the host in another order or more than once"); // Nothing more: no key is written twice. host.set_read_timeout(Some(Duration::from_millis(150))).unwrap(); - loop { - match crate::terminal_host_protocol::read_frame( - &mut host, - crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, - ) { - Ok(Some(frame)) => assert_ne!( - frame.kind, - MessageKind::Input, - "a key reached the host twice: {:?}", - frame.payload - ), - Ok(None) | Err(_) => break, - } + // Ends at the read timeout (nothing more) or at end of stream. + while let Ok(Some(frame)) = crate::terminal_host_protocol::read_frame( + &mut host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) { + assert_ne!( + frame.kind, + MessageKind::Input, + "a key reached the host twice: {:?}", + frame.payload + ); } } From 5586fb82d24550beb5e284e65b1d1f940a09cebf Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:08:20 +0000 Subject: [PATCH 33/42] fix(cmux-tui-core): a failed reconnect completion on a failed link also makes the input port refuse (#34544 review 4) A retry that installed a replacement and then failed to complete it (terminal_host_reconnected false) shuts the replacement down while the link stays Failed; the route was still the shut replacement, so the port took keys and only logged them. That path now publishes InputRoute::Failed under the runtime lock too, as the Failed transition does. Not driven by a test: the hosted fixture's live reader thread reacts to the disconnect itself; the route behavior is covered by a_key_to_a_failed_link_is_refused_until_a_live_route_is_published. --- .../src/surface/hosted_reader/reconnect.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs index 21a934155076..01b698917d51 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs @@ -273,10 +273,20 @@ impl HostedReader { replacement_snapshot.kitty_state.limits, ) { replacement_control_responses.fail_all(); - if let PtyRuntime::Hosted(host) = &*pty.runtime.lock().unwrap() - && host.identity() == identity { - host.disconnect(); + let runtime = pty.runtime.lock().unwrap(); + if let PtyRuntime::Hosted(host) = &*runtime + && host.identity() == identity + { + host.disconnect(); + // A `failed` link stays failed (below): the input port refuses until the next install, as a + // write to the shut replacement fails now (#34544 review 4). + if pty.host_connection_state.load(Ordering::Acquire) + == TerminalHostConnectionState::Failed as u8 + { + pty.input_route_failed(&runtime); + } + } } // A `failed` link being retried stays failed until it connects. let was = pty From 8b9f34a9be86c47326da1f90cb1cf7f71e938803 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:23:20 +0000 Subject: [PATCH 34/42] test(cmux-tui-core): a frame sent after a key the input port holds must reach the host after it (R-DAEMON-STATE S1.6b red) The port counts the frames it took and finished and offers PortOrder::barrier; nothing calls it yet, so a direct frame (here an Activate under the runtime lock, as viewer sizes and the other direct writers send) passes a key the paused port still holds: expected red in a_frame_sent_after_a_key_the_port_holds_reaches_the_host_after_it. --- .../cmux-tui-core/src/surface/input_port.rs | 107 +++++++++++++++++- .../cmux-tui-core/src/surface/tests/hosted.rs | 42 +++++++ 2 files changed, 146 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index 6f56cfe7650a..1bb777bab17a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -103,12 +103,52 @@ struct PortState { queued_bytes: usize, writer_running: bool, closed: bool, + /// Frames the port took, and frames it finished (written, failed or dropped): a direct writer of the host + /// connection waits until `finished` reaches `accepted` as it read it ([`PortOrder::barrier`], S1.6b). + accepted: u64, + finished: u64, + /// The writer waits for a reconnect: a barrier does not wait then; the connection it would write to is gone, and + /// the install it waits for may need the runtime lock that barrier's caller holds. + reconnecting: bool, + /// Tests: the writer waits here before it writes each frame. + #[cfg(test)] + paused: Option>, } #[derive(Default)] struct PortShared { state: Mutex, changed: Condvar, + /// Signalled when `finished` moves, the writer starts waiting for a reconnect, or the port closes. + finished_changed: Condvar, +} + +/// The order of one terminal's input port for the other writers of its host connection (DESIGN 2.6.1, S1.6b): a +/// viewer size, a cell-pixel size, receipted input, a clear history, Kitty limits, a clipboard reply or a terminate +/// issued after a key the port took reaches the host after that key. +#[derive(Clone)] +pub(crate) struct PortOrder(Arc); + +impl PortOrder { + /// Waits until the port finished every frame it took before this call. Returns at once while the port's writer + /// waits for a reconnect or when the port closed, and at `deadline`. Frames taken after the call do not extend + /// the wait. + pub(crate) fn barrier(&self, deadline: Instant) { + let mut state = self.0.state.lock().unwrap_or_else(PoisonError::into_inner); + let target = state.accepted; + while state.finished < target && !state.reconnecting && !state.closed { + let now = Instant::now(); + if now >= deadline { + return; + } + state = self + .0 + .finished_changed + .wait_timeout(state, deadline.saturating_duration_since(now)) + .unwrap_or_else(PoisonError::into_inner) + .0; + } + } } /// One terminal's input port (see the module docs). Closed when the terminal drops. @@ -121,6 +161,7 @@ impl Drop for InputPort { fn drop(&mut self) { self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).closed = true; self.shared.changed.notify_all(); + self.shared.finished_changed.notify_all(); } } @@ -158,11 +199,33 @@ impl InputPort { state.writer_running = true; } state.queued_bytes = total; + state.accepted = state.accepted.saturating_add(1); state.frames.push_back(frame); drop(state); self.shared.changed.notify_one(); Ok(()) } + + /// The port's order for the other writers of the terminal's host connection. + pub(crate) fn order(&self) -> PortOrder { + PortOrder(Arc::clone(&self.shared)) + } + + /// Tests: the writer waits for one message on the returned sender before it writes each frame. + #[cfg(test)] + pub(crate) fn pause_writes_for_test(&self) -> std::sync::mpsc::Sender<()> { + let (resume, paused) = std::sync::mpsc::channel(); + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).paused = Some(paused); + resume + } +} + +/// One frame of the port finished (written, failed or dropped): barriers waiting for it may go. +fn finish_frames(shared: &PortShared, count: u64) { + let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); + state.finished = state.finished.saturating_add(count).min(state.accepted); + drop(state); + shared.finished_changed.notify_all(); } /// The writer: the next frame in order, or `None` when the port closed or stayed idle for @@ -204,7 +267,11 @@ impl Drop for WriterUnwind<'_> { state.frames.clear(); state.queued_bytes = 0; state.writer_running = false; + // Dropped frames and the one it lost are finished: no barrier waits for them. + state.finished = state.accepted; + state.reconnecting = false; drop(state); + self.0.finished_changed.notify_all(); eprintln!( "cmux-tui: terminal input port writer stopped by a panic; {dropped} queued writes dropped" ); @@ -214,9 +281,22 @@ impl Drop for WriterUnwind<'_> { fn run_writer(shared: &PortShared, terminal: &Weak) { let _unwind = WriterUnwind(shared); while let Some(frame) = next_frame(shared) { + #[cfg(test)] + { + let paused = shared.state.lock().unwrap_or_else(PoisonError::into_inner).paused.take(); + if let Some(paused) = paused { + let _ = paused.recv(); + shared.state.lock().unwrap_or_else(PoisonError::into_inner).paused = Some(paused); + } + } // The terminal is gone: its port closed, and nothing is left to write to. - let Some(runtime) = terminal.upgrade() else { return }; - if let Err(error) = write_with_reconnect(&runtime, frame.kind, &frame.bytes) { + let Some(runtime) = terminal.upgrade() else { + finish_frames(shared, 1); + return; + }; + let written = write_with_reconnect(shared, &runtime, frame.kind, &frame.bytes); + finish_frames(shared, 1); + if let Err(error) = written { eprintln!( "cmux-tui: terminal input not written ({} bytes, already accepted): {error}", frame.bytes.len() @@ -225,9 +305,27 @@ fn run_writer(shared: &PortShared, terminal: &Weak) { } } +/// Marks the writer waiting for a reconnect while it lives, so barriers do not wait for it meanwhile. +struct Reconnecting<'a>(&'a PortShared); + +impl<'a> Reconnecting<'a> { + fn start(shared: &'a PortShared) -> Self { + shared.state.lock().unwrap_or_else(PoisonError::into_inner).reconnecting = true; + shared.finished_changed.notify_all(); + Self(shared) + } +} + +impl Drop for Reconnecting<'_> { + fn drop(&mut self) { + self.0.state.lock().unwrap_or_else(PoisonError::into_inner).reconnecting = false; + } +} + /// One frame on the terminal's current host connection, across a lost connection: the rule of /// `Surface::send_hosted_input`, on the port's thread, reading the published route instead of the runtime lock. fn write_with_reconnect( + shared: &PortShared, runtime: &PtyTerminalRuntime, kind: MessageKind, bytes: &[u8], @@ -275,7 +373,10 @@ fn write_with_reconnect( ) || runtime.reader_completion.is_complete() || runtime.owner_detaching.load(Ordering::Acquire) || Instant::now() >= deadline - || runtime.stream_progress.wait_for_change(observed, deadline).is_none() + || { + let _reconnecting = Reconnecting::start(shared); + runtime.stream_progress.wait_for_change(observed, deadline).is_none() + } { return Err(error); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index 0d95f635b3d9..9796c2486a51 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -226,6 +226,48 @@ fn a_key_to_a_failed_link_is_refused_until_a_live_route_is_published() { assert_eq!(frame.payload, b"y", "the refused key must not reach the host"); } +/// R-DAEMON-STATE S1.6b: a frame another writer sends to a hosted terminal's host connection (a viewer size, cell +/// pixels, receipted input, a clear history, Kitty limits, a clipboard reply, a terminate) after a key the input port +/// took reaches the host after that key, though the port has not written it yet. +#[cfg(unix)] +#[test] +fn a_frame_sent_after_a_key_the_port_holds_reaches_the_host_after_it() { + let (_mux, surface, mut host) = hosted_typing_surface("hosted-input-port-order-barrier", true); + let pty = surface.as_pty().expect("a terminal"); + let resume = pty.input_port.pause_writes_for_test(); + surface.write_bytes(b"k").expect("the port took the key"); + std::thread::scope(|scope| { + let direct = scope.spawn(|| { + let runtime = pty.runtime.lock().unwrap(); + let PtyRuntime::Hosted(attachment) = &*runtime else { + panic!("the fixture's terminal is hosted"); + }; + attachment.send(MessageKind::Activate, &[]).expect("the direct frame was written"); + }); + // The direct sender is now inside its send, while the port holds the key. + std::thread::sleep(Duration::from_millis(200)); + resume.send(()).expect("the port's writer waits"); + direct.join().unwrap(); + }); + drop(resume); + let mut kinds = Vec::new(); + while !(kinds.contains(&MessageKind::Input) && kinds.contains(&MessageKind::Activate)) { + let frame = crate::terminal_host_protocol::read_frame( + &mut host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) + .expect("the host reads both frames") + .expect("the host connection stays open"); + kinds.push(frame.kind); + } + let key_at = kinds.iter().position(|kind| *kind == MessageKind::Input); + let frame_at = kinds.iter().position(|kind| *kind == MessageKind::Activate); + assert!( + matches!((key_at, frame_at), (Some(key), Some(frame)) if key < frame), + "a frame sent after a key the port held passed it: {kinds:?}" + ); +} + /// Keys to a hosted terminal on the input port reach the host once each, in the order they were typed (the port's /// one writer thread), and a key the port took returns before the host read anything (#34544 review). #[cfg(unix)] From 91af73e2666ff6355b383bae0e4f4b958a568cf5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:23:42 +0000 Subject: [PATCH 35/42] fix(cmux-tui-core): every other frame to a hosted terminal's host goes after the keys its input port took (R-DAEMON-STATE S1.6b order) HostAttachment gets the terminal's PortOrder (set by the hosted constructor and replace_runtime, so it follows every connection change); each frame it writes besides the port's own (send, receipted input, viewer size, cell pixels, Kitty limits, control requests such as clear history and terminate, and the clipboard replier's replies) first waits until the port finished every frame it had taken (PortOrder::barrier, at most 2 s, as one frame write may block). The wait returns at once while the port's writer waits for a reconnect, since the install it waits for may need the runtime lock the caller holds. With the switch off the port takes nothing and the wait is one uncontended lock. The 9 test literals of HostAttachment take input_order: None. Test: a_frame_sent_after_a_key_the_port_holds_reaches_the_host_after_it (red at 8b9f34a9be8). --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 5 ++- .../src/surface/terminal_runtime.rs | 4 +++ .../shared/attachment.rs | 35 ++++++++++++++++--- .../shared/attachment/connect.rs | 1 + .../shared/attachment/pending_control.rs | 2 +- .../shared/attachment/terminate.rs | 2 +- .../shared/clipboard_read.rs | 8 +++++ .../src/terminal_host_runtime/unix/tests.rs | 1 + .../unix/tests/attach_protocol.rs | 3 ++ .../unix/tests/input_records.rs | 3 ++ .../unix/tests/launch_codec.rs | 1 + 11 files changed, 58 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 7714f6d264b6..4da5708a2314 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -778,6 +778,9 @@ impl Surface { let frame_requests = frame_producer::FrameRequests::new(); #[cfg(test)] let frame_producer_before_upgrade = Arc::new(Mutex::new(None)); + // The attachment's other frames go after the keys the port took (S1.6b). + let port = input_port::InputPort::default(); + attachment.set_input_order(port.order()); let surface = Arc::new(Surface::Pty(PtySurface { meta: SurfaceMeta { id, @@ -814,7 +817,7 @@ impl Surface { input_route: arc_swap::ArcSwapOption::new(Some(Arc::new( input_port::InputRoute::Hosted(attachment.input_route()), ))), - input_port: Default::default(), + input_port: port, runtime: RankedMutex::new( LockRank::Runtime, "pty.runtime", diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index 2d6cccf5f6f6..dc2cecf12bae 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -69,6 +69,10 @@ impl PtyTerminalRuntime { #[cfg(any(unix, windows))] pub(super) fn replace_runtime(&self, runtime: &mut PtyRuntime, next: PtyRuntime) { *runtime = next; + // The new connection's other frames go after the keys the port took (S1.6b). + if let PtyRuntime::Hosted(host) = runtime { + host.set_input_order(self.input_port.order()); + } self.input_route.store(Self::input_route_of(runtime)); } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs index 01f1405713cc..7ec287e730c3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs @@ -13,6 +13,10 @@ use std::time::{Duration, Instant}; use anyhow::Context; +/// The longest a frame other than the input port's own waits for the port's earlier keys (S1.6b): as long as one +/// frame write may block on a host socket. +const INPUT_ORDER_WAIT: Duration = Duration::from_secs(2); + pub(crate) use super::super::sys::SpawnedHostProcess; use super::super::sys::{HostStream, PtyCustody, connect_with_retry}; use super::super::*; @@ -131,6 +135,9 @@ pub struct HostAttachment { pub(crate) launch_activation_pending: bool, /// The owner's copy of this host's PTY master (`pty_custody.rs`). pub(crate) pty_custody: Option, + /// The terminal's input port order (R-DAEMON-STATE S1.6b): every other frame this attachment writes waits until + /// the port wrote the keys it took before, so no frame passes a key. Set by the terminal that owns the attachment. + pub(crate) input_order: Option, } impl std::fmt::Debug for HostAttachment { @@ -189,9 +196,29 @@ impl HostAttachment { } pub fn send(&self, kind: MessageKind, payload: &[u8]) -> std::io::Result<()> { + self.wait_input_order(); send_host_frame(&self.writer, self.protocol_version, kind, payload) } + /// The terminal's input port order for this attachment's frames (S1.6b). + pub(crate) fn set_input_order(&mut self, order: crate::surface::input_port::PortOrder) { + self.input_order = Some(order); + } + + /// Before a frame other than the port's own: the keys the port took before are written first (S1.6b). + pub(crate) fn wait_input_order(&self) { + if let Some(order) = &self.input_order { + order + .barrier(Instant::now().checked_add(INPUT_ORDER_WAIT).unwrap_or_else(Instant::now)); + } + } + + /// The writer for a frame other than the port's own, after the port's earlier keys (S1.6b). + fn direct_writer(&self) -> std::sync::MutexGuard<'_, HostStream> { + self.wait_input_order(); + self.writer.lock().recover_poison() + } + /// The route the terminal's input port writes this connection's interactive input on. pub(crate) fn input_route(&self) -> HostInputRoute { HostInputRoute { writer: self.writer.clone(), protocol_version: self.protocol_version } @@ -255,7 +282,7 @@ impl HostAttachment { frame.version = self.protocol_version; frame.request_id = request_id; let write_result = { - let mut writer = self.writer.lock().recover_poison(); + let mut writer = self.direct_writer(); let result = write_frame(&mut *writer, &frame).map_err(protocol_io_error); if result.is_err() { let _ = writer.shutdown(std::net::Shutdown::Both); @@ -329,7 +356,7 @@ impl HostAttachment { let mut frame = Frame::new(MessageKind::ViewerSize, payload); frame.version = self.protocol_version; frame.request_id = request_id; - let written = write_frame(&mut *self.writer.lock().recover_poison(), &frame).is_ok(); + let written = write_frame(&mut *self.direct_writer(), &frame).is_ok(); drop(viewer_size); if !written { self.control_responses.waiters.lock().recover_poison().remove(&request_id); @@ -401,7 +428,7 @@ impl HostAttachment { frame.version = self.protocol_version; frame.request_id = request_id; let write_result = { - let mut writer = self.writer.lock().recover_poison(); + let mut writer = self.direct_writer(); write_frame(&mut *writer, &frame).map_err(protocol_io_error) }; if let Err(error) = write_result { @@ -481,7 +508,7 @@ impl HostAttachment { request.version = self.protocol_version; request.request_id = request_id; let write_result = { - let mut writer = self.writer.lock().recover_poison(); + let mut writer = self.direct_writer(); write_frame(&mut *writer, &request).map_err(protocol_io_error) }; if let Err(error) = write_result { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/connect.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/connect.rs index 4da31b183b93..1b90d336c2e1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/connect.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/connect.rs @@ -282,6 +282,7 @@ pub(crate) fn connect_record_at_version( launch_process: None, launch_activation_pending, pty_custody: None, + input_order: None, }; attachment.release_viewer_size()?; Ok(attachment) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs index 85c971253268..64f4e6931fe1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs @@ -53,7 +53,7 @@ impl HostAttachment { frame.version = self.protocol_version; frame.request_id = request_id; let write_result = { - let mut writer = self.writer.lock().recover_poison(); + let mut writer = self.direct_writer(); let result = write_frame(&mut *writer, &frame).map_err(protocol_io_error); if result.is_err() { let _ = writer.shutdown(std::net::Shutdown::Both); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/terminate.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/terminate.rs index 820442d0313b..9dc2d29c780c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/terminate.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/terminate.rs @@ -46,7 +46,7 @@ impl HostAttachment { request.version = self.protocol_version; request.request_id = request_id; { - let mut writer = self.writer.lock().recover_poison(); + let mut writer = self.direct_writer(); write_frame(&mut *writer, &request).map_err(protocol_io_error)?; } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs index 3a4309b65957..ea8cb46241f0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs @@ -623,6 +623,7 @@ impl HostAttachment { writer: Arc::downgrade(&self.writer), responses: Arc::downgrade(&self.control_responses), protocol_version: self.protocol_version, + input_order: self.input_order.clone(), } } } @@ -636,6 +637,8 @@ pub(crate) struct ClipboardReplier { pub(crate) writer: Weak>, pub(crate) responses: Weak, pub(crate) protocol_version: u16, + /// The terminal's input port order: the reply goes after the keys the port took before it (S1.6b). + pub(crate) input_order: Option, } impl ClipboardReplier { @@ -651,6 +654,11 @@ impl ClipboardReplier { return Ok(false); } let reply = encode_clipboard_read_reply(token, text); + if let Some(order) = &self.input_order { + order.barrier( + Instant::now().checked_add(Duration::from_secs(2)).unwrap_or_else(Instant::now), + ); + } send_host_frame(&writer, self.protocol_version, MessageKind::ClipboardReadReply, &reply)?; Ok(true) } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests.rs index 30582c814216..8f08bfc57905 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests.rs @@ -314,6 +314,7 @@ pub(crate) fn input_ack_surface_fixture() -> (HostAttachment, UnixStream) { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; (attachment, host) } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/attach_protocol.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/attach_protocol.rs index 154dc30f0173..77de7eebdccc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/attach_protocol.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/attach_protocol.rs @@ -97,6 +97,7 @@ fn timed_out_cell_pixel_ack_reconciles_when_the_response_arrives_late() { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; let (release_ack_tx, release_ack_rx) = std::sync::mpsc::channel(); let resolver = { @@ -214,6 +215,7 @@ fn detach_fence_reports_a_delayed_receipt_after_output_as_a_failure() { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; let (output_queued, output_seen) = sync_channel(1); let (release_ack, ack_release) = sync_channel(1); @@ -378,6 +380,7 @@ fn adoption_quota_reconfiguration_finishes_before_snapshot_use() { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; let responder = thread::spawn(move || { let request = read_frame(&mut host, MAX_FRAME_PAYLOAD).unwrap().unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/input_records.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/input_records.rs index f32cf0a5c60e..fd320def63c4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/input_records.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/input_records.rs @@ -39,6 +39,7 @@ fn receipted_input_never_reaches_a_legacy_host_without_ack_support() { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; let error = match attachment.begin_input_confirmed(b"must-not-send") { @@ -305,6 +306,7 @@ fn terminate_waits_for_the_authoritative_host_receipt() { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; let responder = thread::spawn(move || { let request = read_frame(&mut host, MAX_FRAME_PAYLOAD).unwrap().unwrap(); @@ -358,6 +360,7 @@ fn clear_history_control_write_failure_after_header_is_ambiguous() { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; let peer = thread::spawn(move || { let mut header = [0; crate::terminal_host_protocol::HEADER_LEN]; diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/launch_codec.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/launch_codec.rs index 74961f134155..4288a3382df5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/launch_codec.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/unix/tests/launch_codec.rs @@ -519,6 +519,7 @@ fn clear_history_ack_preserves_known_not_delivered_failure() { launch_process: None, launch_activation_pending: false, pty_custody: None, + input_order: None, }; let responder = thread::spawn(move || { let request = read_frame(&mut host, MAX_FRAME_PAYLOAD).unwrap().unwrap(); From 189ba5886e5bce2de7a8d57d4e23b34cc3c51e5b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:49:44 +0000 Subject: [PATCH 36/42] test(cmux-tui-core): an automation write answers after the input port wrote it, with its frame's error (R-DAEMON-STATE S1.6b red) On the lane every hosted write answered when the port took the bytes, so the send handler, the creating-input flush (its give-back) and the resource input effect (its indeterminate mark) never saw a write error. Expected red: an_automation_write_answers_after_the_port_wrote_it (it answers while the port is paused) and an_automation_write_gets_the_error_of_its_frame (ok for bytes never written). --- .../cmux-tui-core/src/surface/tests/hosted.rs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index 9796c2486a51..b2f067601524 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -268,6 +268,55 @@ fn a_frame_sent_after_a_key_the_port_holds_reaches_the_host_after_it() { ); } +/// R-DAEMON-STATE S1.6b: a write that is not the person's key (a script's `cmux send`, a resource input effect, the +/// creating-input flush) answers when the input port wrote it, as the old path answered after its socket write, so +/// it never answers ok for bytes the port has not written. +#[cfg(unix)] +#[test] +fn an_automation_write_answers_after_the_port_wrote_it() { + let (_mux, surface, mut host) = + hosted_typing_surface("hosted-input-port-automation-reply", true); + let pty = surface.as_pty().expect("a terminal"); + let resume = pty.input_port.pause_writes_for_test(); + std::thread::scope(|scope| { + let writing = scope.spawn(|| surface.write_bytes(b"a")); + std::thread::sleep(Duration::from_millis(200)); + let answered_early = writing.is_finished(); + resume.send(()).expect("the port's writer waits"); + writing.join().unwrap().expect("the write"); + assert!(!answered_early, "an automation write answered before the port wrote it"); + }); + let frame = crate::terminal_host_protocol::read_frame( + &mut host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) + .expect("the host reads the write") + .expect("the host connection stays open"); + assert_eq!((frame.kind, frame.payload.as_slice()), (MessageKind::Input, &b"a"[..])); +} + +/// R-DAEMON-STATE S1.6b: a write that is not the person's key gets the error of its own frame (here the reader +/// shut the connection down after the port took the bytes), as the old path's write failed, so `cmux send` exits +/// non-zero and a resource input effect is marked failed. +#[cfg(unix)] +#[test] +fn an_automation_write_gets_the_error_of_its_frame() { + let (_mux, surface, _host) = hosted_typing_surface("hosted-input-port-automation-error", true); + let pty = surface.as_pty().expect("a terminal"); + let resume = pty.input_port.pause_writes_for_test(); + std::thread::scope(|scope| { + let writing = scope.spawn(|| surface.write_bytes(b"b")); + std::thread::sleep(Duration::from_millis(100)); + pty.input_route.store(Some(Arc::new(input_port::InputRoute::Failed))); + resume.send(()).expect("the port's writer waits"); + let error = writing + .join() + .unwrap() + .expect_err("an automation write the port could not write answered ok"); + assert_eq!(error.kind(), std::io::ErrorKind::NotConnected, "{error}"); + }); +} + /// Keys to a hosted terminal on the input port reach the host once each, in the order they were typed (the port's /// one writer thread), and a key the port took returns before the host read anything (#34544 review). #[cfg(unix)] From 9afa30090daf617bef787145947edbfc5dfb3f1e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:57:03 +0000 Subject: [PATCH 37/42] fix(cmux-tui-core): on the input lane only the person's keys answer before the write; every other write answers with its frame's result (R-DAEMON-STATE S1.6b) S1.6a answered every hosted write when the port took it, so the send handler for scripts, the creating-input flush (its give-back), the resource input effect (its indeterminate mark) and the image paste (its reservation) never saw a write error. Now Surface::write_bytes and write_paste answer when the port wrote the frame (InputReply::Written: the port reports each frame's result on a channel, bounded at 15 s); only the person's key (the send and send-key handlers, when the client is attached to the terminal it types into) passes InputReply::Accepted and returns at once. The port also keeps one reconnect wait for its queue: frames queued behind a host that does not come back fail together, not 5 s each. The give-back notice of the plan is dropped (DESIGN 2.6.1: the app reads it only on its control connection, and it never retried a failed key). Tests: an_automation_write_answers_after_the_port_wrote_it and an_automation_write_gets_the_error_of_its_frame (red at 189ba5886e5); the_persons_keys_answer_when_taken_and_other_writes_when_written; the person-key tests pass InputReply::Accepted. --- .../src/server/cmd_terminal_io.rs | 30 +++++---- .../src/server/tests/rds_input_lane.rs | 28 +++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 1 + .../crates/cmux-tui-core/src/surface/input.rs | 63 ++++++++++++++++--- .../cmux-tui-core/src/surface/input_port.rs | 47 +++++++++++--- .../cmux-tui-core/src/surface/tests/hosted.rs | 7 ++- 6 files changed, 145 insertions(+), 31 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs b/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs index 40367569e9d2..9ff60355935e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/cmd_terminal_io.rs @@ -4,6 +4,7 @@ //! `Command` arm of `handle_command_with_cancellation`. use crate::Rgb; +use crate::surface::InputReply; use super::styled_run_json; use base64::Engine; @@ -126,12 +127,16 @@ pub(super) struct SendParams { /// A key from a client that can type into its terminal (attached, not only through read-only views) is the person's: /// its connection's threads then run at user-interactive QoS (server/input_qos.rs, TYPING FIX 2 F7). A read-only /// viewer's keys are automation's and keep the default QoS. -fn input_surface( +/// +/// Also how the write answers on the input lane (R-DAEMON-STATE S1.6b): the person's key returns when the terminal's +/// input port took it ([`InputReply::Accepted`]); anyone else's write answers when the port wrote it, with its error +/// ([`InputReply::Written`]), as the old path answered. Off the lane the write answers after the write either way. +pub(super) fn input_surface( mux: &Arc, client: u64, writer: &MessageWriter, id: SurfaceId, -) -> anyhow::Result> { +) -> anyhow::Result<(Arc, InputReply)> { if mux.control_clients.input_lane.on() { // R-DAEMON-STATE stage S1: the session's published routes give the terminal `id` routes to now (a respawn, // close or move included) and the client registry's published facts say whether the client types into it. @@ -139,21 +144,22 @@ fn input_surface( let surface = get_surface(mux, id)?; if mux.control_clients.types_into(client, id) { writer.input_lane.mark_person_input(); + return Ok((surface, InputReply::Accepted)); } - return Ok(surface); + return Ok((surface, InputReply::Written)); } if let Some((surface, can_type)) = mux.control_clients.attached_input_surface(client, id) { if can_type { writer.input_lane.mark_person_input(); } - return Ok(surface); + return Ok((surface, InputReply::Written)); } let generation = crate::route_table::route_generation(id); let surface = get_surface(mux, id)?; if mux.control_clients.remember_attached_input(client, id, &surface, generation) { writer.input_lane.mark_person_input(); } - Ok(surface) + Ok((surface, InputReply::Written)) } pub(super) fn send( @@ -193,7 +199,7 @@ pub(super) fn send( } _ => anyhow::bail!("bad request: send takes exactly one of surface and terminal_id"), }; - let surface = input_surface(mux, client, writer, surface)?; + let (surface, reply) = input_surface(mux, client, writer, surface)?; require_pty(&surface)?; // Bytes queued for this terminal while it was being created go first. mux.wait_for_creating_input_flush(&surface); @@ -235,10 +241,10 @@ pub(super) fn send( if let Some(b64) = bytes { payload.extend(base64::engine::general_purpose::STANDARD.decode(b64)?); } - surface.write_paste(&payload)?; + surface.write_paste_replying(&payload, reply)?; } else { if let Some(text) = text { - surface.write_bytes(text.as_bytes())?; + surface.write_bytes_replying(text.as_bytes(), reply)?; } if let Some(b64) = bytes { let raw = base64::engine::general_purpose::STANDARD.decode(b64)?; @@ -247,7 +253,7 @@ pub(super) fn send( flow.unwrap_or(0), surface.flow_terminal_id(), ); - surface.write_bytes(&raw)?; + surface.write_bytes_replying(&raw, reply)?; cmux_flow_trace::hop( "daemon.send.written", flow.unwrap_or(0), @@ -259,7 +265,7 @@ pub(super) fn send( // Return as its own key: a full-screen agent editor reads a pasted newline as a line // break, not a submit. // The text is already in: a failed Return must not read as "nothing was written". - surface.write_bytes(b"\r").map_err(|error| { + surface.write_bytes_replying(b"\r", reply).map_err(|error| { anyhow::anyhow!("submit failed after the text was written: {error}") })?; } @@ -471,7 +477,7 @@ pub(super) fn send_key( keys: Vec, ) -> anyhow::Result { // The same kept terminal and route rules as `send`: a named key takes no session lock (TYPING FIX 2). - let surface = input_surface(mux, client, writer, surface)?; + let (surface, reply) = input_surface(mux, client, writer, surface)?; require_pty(&surface).map_err(|_| anyhow::anyhow!("surface does not support key input"))?; if keys.is_empty() { anyhow::bail!("bad request: keys must be non-empty"); @@ -490,7 +496,7 @@ pub(super) fn send_key( Ok::<(), anyhow::Error>(()) })??; mux.wait_for_creating_input_flush(&surface); - surface.write_bytes(&encoded)?; + surface.write_bytes_replying(&encoded, reply)?; mux.note_terminal_input(&surface, client); Ok(json!({})) } diff --git a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs index b52020b5df90..0194f2f1fdcc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server/tests/rds_input_lane.rs @@ -426,6 +426,34 @@ fn a_settled_size_state_read_takes_the_session_lock_once_with_the_switch_off() { } } +/// R-DAEMON-STATE S1.6b: on the lane the person's key (a connection attached to the terminal it types into) answers +/// when the input port took it; any other client's write (a script's `cmux send`, MCP) answers when the port wrote it, +/// with its error, as the old path answered. +#[test] +fn the_persons_keys_answer_when_taken_and_other_writes_when_written() { + let connection = TypingConnection::open(); + let (_, reply) = cmd_terminal_io::input_surface( + &connection.mux, + connection.client, + &connection.writer, + connection.surface, + ) + .unwrap(); + assert_eq!(reply, crate::surface::InputReply::Accepted, "the person's key waits for its write"); + let (script_writer, _script_outbound) = captured_writer(); + let script = + connection.mux.control_clients.register(ClientTransport::Unix, script_writer.clone()); + let (_, reply) = + cmd_terminal_io::input_surface(&connection.mux, script, &script_writer, connection.surface) + .unwrap(); + assert_eq!( + reply, + crate::surface::InputReply::Written, + "a script's write answers before it is written" + ); + connection.close(); +} + /// The skipped note must never keep the grid from a view: under the default policy (the newest view sets the grid), /// another view's focus click takes the grid, and the next key from the first view takes it back, on both arms. #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 4da5708a2314..31485adbe130 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -66,6 +66,7 @@ mod hosted_stager; pub(crate) mod input_port; #[cfg(any(unix, windows))] use hosted_stager::{HostedFrameStager, HostedTransition}; +pub(crate) use input_port::InputReply; #[cfg(test)] use options::child_term_for; use options::configure_agent_browser_session; diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index 0481cfa86fda..692952e03021 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -12,6 +12,12 @@ use super::*; /// the terminal's input port waits instead (`surface::input_port`). pub(super) const HOST_INPUT_RECONNECT_WAIT: Duration = Duration::from_secs(5); +/// The longest a write that waits for its frame ([`InputReply::Written`]) waits for the terminal's input port: one +/// reconnect wait for the queue ahead of it, a few blocked socket writes, and slack. Past it the write answers an +/// error and the bytes may still be written. +#[cfg(any(unix, windows))] +const INPUT_WRITE_REPLY_WAIT: Duration = Duration::from_secs(15); + /// Whether a host send failed because the host connection is gone (the /// host died or its stream broke), not because the frame was refused. #[cfg(any(unix, windows))] @@ -26,8 +32,19 @@ pub(super) fn host_connection_lost(error: &std::io::Error) -> bool { } impl Surface { - /// Write input bytes to the PTY child. + /// Write input bytes to the PTY child. On the input lane (R-DAEMON-STATE S1) a hosted terminal's write answers + /// when its input port wrote it ([`InputReply::Written`]), with the write's error as before. pub fn write_bytes(&self, bytes: &[u8]) -> std::io::Result<()> { + self.write_bytes_replying(bytes, InputReply::Written) + } + + /// [`Self::write_bytes`] answering as `reply` says on the input lane: the person's keys pass + /// [`InputReply::Accepted`] and return when the terminal's input port took them. + pub(crate) fn write_bytes_replying( + &self, + bytes: &[u8], + reply: InputReply, + ) -> std::io::Result<()> { let Some(pty) = self.as_pty() else { return Err(std::io::Error::new( std::io::ErrorKind::Unsupported, @@ -35,7 +52,7 @@ impl Surface { )); }; #[cfg(any(unix, windows))] - if let Some(result) = Self::enqueue_on_input_port(pty, MessageKind::Input, bytes) { + if let Some(result) = Self::enqueue_on_input_port(pty, MessageKind::Input, bytes, reply) { return result; } #[cfg(any(unix, windows))] @@ -69,6 +86,7 @@ impl Surface { pty: &PtySurface, kind: MessageKind, bytes: &[u8], + reply: InputReply, ) -> Option> { if !pty.mux.upgrade().is_some_and(|mux| mux.control_clients.input_lane.on()) { return None; @@ -89,9 +107,28 @@ impl Surface { "the terminal's host connection ended; input not written", ))) } - Some(input_port::InputRoute::Hosted(_)) => { - Some(pty.input_port.enqueue(&pty.terminal, kind, bytes).map_err(Into::into)) - } + Some(input_port::InputRoute::Hosted(_)) => Some(match reply { + InputReply::Accepted => { + pty.input_port.enqueue(&pty.terminal, kind, bytes, None).map_err(Into::into) + } + InputReply::Written => { + let (done, written) = std::sync::mpsc::sync_channel(1); + pty.input_port + .enqueue(&pty.terminal, kind, bytes, Some(done)) + .map_err(std::io::Error::from) + .and_then(|()| match written.recv_timeout(INPUT_WRITE_REPLY_WAIT) { + Ok(result) => result, + Err(RecvTimeoutError::Timeout) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal input still waits in the terminal's input port", + )), + Err(RecvTimeoutError::Disconnected) => Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "the terminal's input port closed before it wrote the input", + )), + }) + } + }), } } @@ -191,7 +228,16 @@ impl Surface { /// Write a protocol input payload, conditionally applying bracketed-paste /// markers from a terminal-mode snapshot taken before the PTY write. pub fn write_paste(&self, bytes: &[u8]) -> std::io::Result<()> { - self.write_paste_with_timeout(bytes, None) + self.write_paste_with_timeout(bytes, None, InputReply::Written) + } + + /// [`Self::write_paste`] answering as `reply` says on the input lane ([`Self::write_bytes_replying`]). + pub(crate) fn write_paste_replying( + &self, + bytes: &[u8], + reply: InputReply, + ) -> std::io::Result<()> { + self.write_paste_with_timeout(bytes, None, reply) } /// Write a paste with a bounded local PTY write for daemon-owned uploads. @@ -201,13 +247,14 @@ impl Surface { /// polled until the same two-second bound, so a full PTY cannot retain an /// image-paste reservation indefinitely. pub(crate) fn write_paste_bounded(&self, bytes: &[u8]) -> std::io::Result<()> { - self.write_paste_with_timeout(bytes, Some(LOCAL_PASTE_WRITE_TIMEOUT)) + self.write_paste_with_timeout(bytes, Some(LOCAL_PASTE_WRITE_TIMEOUT), InputReply::Written) } pub(super) fn write_paste_with_timeout( &self, bytes: &[u8], timeout: Option, + reply: InputReply, ) -> std::io::Result<()> { let Some(pty) = self.as_pty() else { return Err(std::io::Error::new( @@ -221,7 +268,7 @@ impl Surface { // Keep-on-exit terminals accept and drop paste input the same way // as keystrokes: the final screen is read-only, not broken. #[cfg(any(unix, windows))] - if let Some(result) = Self::enqueue_on_input_port(pty, MessageKind::Paste, bytes) { + if let Some(result) = Self::enqueue_on_input_port(pty, MessageKind::Paste, bytes, reply) { return result; } #[cfg(any(unix, windows))] diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index 1bb777bab17a..ae6f65848cd1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -85,9 +85,25 @@ impl From for std::io::Error { } } +/// Where the port reports one frame's write to a writer that waits for it ([`InputReply::Written`]). +pub(crate) type FrameDone = std::sync::mpsc::SyncSender>; + +/// What a write to a hosted terminal's input port answers (S1.6b): the person's keys return when the port took them; +/// every other writer (a script's `send`, a resource input effect, the creating-input flush, an image paste) waits +/// for its frame's write, so its error reaches it as on the old path. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum InputReply { + /// The port took the bytes; nothing waits (the person's keys). + Accepted, + /// The port wrote the bytes, or the write's error. + Written, +} + struct PortFrame { kind: MessageKind, bytes: Vec, + /// The waiting writer of an [`InputReply::Written`] write; dropped unanswered when the frame is dropped. + done: Option, } impl PortFrame { @@ -173,12 +189,13 @@ impl InputPort { terminal: &Arc, kind: MessageKind, bytes: &[u8], + done: Option, ) -> Result<(), InputRefused> { if bytes.len() > MAX_FRAME_PAYLOAD { return Err(InputRefused::TooLarge); } // Copied before the lock, so another write to this terminal never waits for this one's copy. - let frame = PortFrame { kind, bytes: bytes.to_vec() }; + let frame = PortFrame { kind, bytes: bytes.to_vec(), done }; let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); let Some(total) = state .queued_bytes @@ -280,6 +297,9 @@ impl Drop for WriterUnwind<'_> { fn run_writer(shared: &PortShared, terminal: &Weak) { let _unwind = WriterUnwind(shared); + // One reconnect wait for the queue: set by the first frame that finds the connection lost, cleared by the next + // write that succeeds, so frames queued behind a host that never comes back fail together, not 5 s each. + let mut reconnect_until = None; while let Some(frame) = next_frame(shared) { #[cfg(test)] { @@ -294,13 +314,19 @@ fn run_writer(shared: &PortShared, terminal: &Weak) { finish_frames(shared, 1); return; }; - let written = write_with_reconnect(shared, &runtime, frame.kind, &frame.bytes); + let written = + write_with_reconnect(shared, &runtime, frame.kind, &frame.bytes, &mut reconnect_until); finish_frames(shared, 1); - if let Err(error) = written { - eprintln!( + match (written, frame.done) { + // The waiting writer answers its request with the result. + (written, Some(done)) => { + let _ = done.send(written); + } + (Err(error), None) => eprintln!( "cmux-tui: terminal input not written ({} bytes, already accepted): {error}", frame.bytes.len() - ); + ), + (Ok(()), None) => {} } } } @@ -329,9 +355,8 @@ fn write_with_reconnect( runtime: &PtyTerminalRuntime, kind: MessageKind, bytes: &[u8], + reconnect_until: &mut Option, ) -> std::io::Result<()> { - let deadline = - Instant::now().checked_add(HOST_INPUT_RECONNECT_WAIT).unwrap_or_else(Instant::now); let mut first_error = None; loop { // Read the revision before the send: an install after it wakes the wait below, so no reconnect is missed. @@ -359,11 +384,17 @@ fn write_with_reconnect( } }; let error = match sent { - Ok(()) => return Ok(()), + Ok(()) => { + *reconnect_until = None; + return Ok(()); + } Err(error) if super::input::host_connection_lost(&error) => error, Err(error) => return Err(error), }; let error = first_error.take().unwrap_or(error); + let deadline = *reconnect_until.get_or_insert_with(|| { + Instant::now().checked_add(HOST_INPUT_RECONNECT_WAIT).unwrap_or_else(Instant::now) + }); let state = TerminalHostConnectionState::from_u8( runtime.host_connection_state.load(Ordering::Acquire), ); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index b2f067601524..cfa58ffe3f62 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -153,7 +153,8 @@ fn key_behind_a_blocked_paste(lane: bool) -> (Duration, Vec) { // Long enough for the paste's writer to be inside its blocked socket write, well inside the stall. std::thread::sleep(Duration::from_millis(150)); let started = Instant::now(); - surface.write_bytes(b"k").expect("the key was accepted"); + // The person's key: it answers when the port took it (on the old path, after its write). + surface.write_bytes_replying(b"k", InputReply::Accepted).expect("the key was accepted"); let took = started.elapsed(); pasting.join().unwrap().expect("the paste was written"); (took, reader.join().unwrap()) @@ -235,7 +236,7 @@ fn a_frame_sent_after_a_key_the_port_holds_reaches_the_host_after_it() { let (_mux, surface, mut host) = hosted_typing_surface("hosted-input-port-order-barrier", true); let pty = surface.as_pty().expect("a terminal"); let resume = pty.input_port.pause_writes_for_test(); - surface.write_bytes(b"k").expect("the port took the key"); + surface.write_bytes_replying(b"k", InputReply::Accepted).expect("the port took the key"); std::thread::scope(|scope| { let direct = scope.spawn(|| { let runtime = pty.runtime.lock().unwrap(); @@ -325,7 +326,7 @@ fn keys_on_the_input_port_reach_the_host_once_each_in_order() { let (_mux, surface, mut host) = hosted_typing_surface("hosted-input-port-order", true); let keys: Vec = (b'a'..=b'z').collect(); for key in &keys { - surface.write_bytes(&[*key]).expect("the port took the key"); + surface.write_bytes_replying(&[*key], InputReply::Accepted).expect("the port took the key"); } let mut written = Vec::new(); while written.len() < keys.len() { From 2133c10ae2f65a8f7dd2cd7092830040eb1faab7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:12:32 +0000 Subject: [PATCH 38/42] test(cmux-tui-core): the input port's reconnect and reply rules from review 1 of #35082 (R-DAEMON-STATE S1.6b red) Red: - a_frame_sent_after_a_replacement_install_waits_for_the_keys_the_port_held (must-fix 1): after replace_runtime and before the reader's stream notify, a direct frame on the new connection passes the key the port held. - a_later_host_loss_waits_for_its_own_reconnect (must-fix 2): the first loss's deadline outlives it, so a later loss's write fails at once. - a_write_whose_wait_ends_before_the_port_takes_it_is_never_written, a_write_whose_wait_ends_during_its_write_may_be_written and the_creating_input_flush_keeps_input_that_may_have_been_written (item 6): a timed-out write stays queued and is written later, and the flush gives back bytes that may still be written. Guards (pass before and after): a_frame_sent_while_the_port_waits_for_a_reconnect_does_not_wait_for_it, writes_queued_behind_a_lost_host_fail_together, receipted_input_sent_after_a_key_the_port_holds_reaches_the_host_after_it. an_automation_write_gets_the_error_of_its_frame waits for the port's count instead of sleeping 100 ms. Test seams: the port's reply and reconnect waits (test overrides), its reconnecting/accepted/queued counts, and the InputMayBeWritten marker with its classifier (nothing produces the marker yet). --- .../crates/cmux-tui-core/src/surface/input.rs | 8 +- .../cmux-tui-core/src/surface/input_port.rs | 82 ++++- .../cmux-tui-core/src/surface/tests/hosted.rs | 295 +++++++++++++++++- 3 files changed, 376 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index 692952e03021..a68338ed09ca 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -12,12 +12,6 @@ use super::*; /// the terminal's input port waits instead (`surface::input_port`). pub(super) const HOST_INPUT_RECONNECT_WAIT: Duration = Duration::from_secs(5); -/// The longest a write that waits for its frame ([`InputReply::Written`]) waits for the terminal's input port: one -/// reconnect wait for the queue ahead of it, a few blocked socket writes, and slack. Past it the write answers an -/// error and the bytes may still be written. -#[cfg(any(unix, windows))] -const INPUT_WRITE_REPLY_WAIT: Duration = Duration::from_secs(15); - /// Whether a host send failed because the host connection is gone (the /// host died or its stream broke), not because the frame was refused. #[cfg(any(unix, windows))] @@ -116,7 +110,7 @@ impl Surface { pty.input_port .enqueue(&pty.terminal, kind, bytes, Some(done)) .map_err(std::io::Error::from) - .and_then(|()| match written.recv_timeout(INPUT_WRITE_REPLY_WAIT) { + .and_then(|()| match written.recv_timeout(pty.input_port.reply_wait()) { Ok(result) => result, Err(RecvTimeoutError::Timeout) => Err(std::io::Error::new( std::io::ErrorKind::TimedOut, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index ae6f65848cd1..04907060eaa7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -46,6 +46,29 @@ const INPUT_PORT_BYTE_CAPACITY: usize = 2 * (MAX_FRAME_PAYLOAD + PORT_FRAME_COST const INPUT_PORT_IDLE_EXIT: Duration = Duration::from_secs(30); /// The writer's stack: it formats and writes one frame at a time. const INPUT_PORT_STACK_BYTES: usize = 256 * 1024; +/// The longest a write that waits for its frame ([`InputReply::Written`]) waits for the terminal's input port: one +/// reconnect wait for the queue ahead of it, a few blocked socket writes, and slack. +pub(super) const INPUT_WRITE_REPLY_WAIT: Duration = Duration::from_secs(15); + +/// The error inside a write's error when its wait ended while the port's writer was writing it: the bytes may still +/// reach the program, so a caller that gives input back must not give these back. +#[derive(Debug)] +pub(crate) struct InputMayBeWritten; + +impl std::fmt::Display for InputMayBeWritten { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str( + "terminal input may have been written: the wait ended during its write to the terminal's host", + ) + } +} + +impl std::error::Error for InputMayBeWritten {} + +/// Whether `error` is the error of a write that may still reach the program ([`InputMayBeWritten`]). +pub(crate) fn input_may_be_written(error: &std::io::Error) -> bool { + error.get_ref().is_some_and(|inner| inner.is::()) +} /// Where a hosted terminal's interactive input goes, published without the runtime lock. pub(crate) enum InputRoute { @@ -129,6 +152,11 @@ struct PortState { /// Tests: the writer waits here before it writes each frame. #[cfg(test)] paused: Option>, + /// Tests: [`INPUT_WRITE_REPLY_WAIT`] and [`HOST_INPUT_RECONNECT_WAIT`] for this port. + #[cfg(test)] + reply_wait: Option, + #[cfg(test)] + reconnect_wait: Option, } #[derive(Default)] @@ -235,6 +263,58 @@ impl InputPort { self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).paused = Some(paused); resume } + + /// How long a write that waits for its frame waits ([`INPUT_WRITE_REPLY_WAIT`]). + pub(crate) fn reply_wait(&self) -> Duration { + #[cfg(test)] + if let Some(wait) = + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).reply_wait + { + return wait; + } + INPUT_WRITE_REPLY_WAIT + } + + /// Tests: [`Self::reply_wait`] for this port. + #[cfg(test)] + pub(crate) fn set_reply_wait_for_test(&self, wait: Duration) { + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).reply_wait = Some(wait); + } + + /// Tests: the reconnect wait ([`HOST_INPUT_RECONNECT_WAIT`]) for this port. + #[cfg(test)] + pub(crate) fn set_reconnect_wait_for_test(&self, wait: Duration) { + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).reconnect_wait = + Some(wait); + } + + /// Tests: whether the writer waits for a reconnect now. + #[cfg(test)] + pub(crate) fn reconnecting_for_test(&self) -> bool { + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).reconnecting + } + + /// Tests: the frames the port took so far. + #[cfg(test)] + pub(crate) fn accepted_for_test(&self) -> u64 { + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).accepted + } + + /// Tests: the frames still queued (not yet taken by the writer). + #[cfg(test)] + pub(crate) fn queued_frames_for_test(&self) -> usize { + self.shared.state.lock().unwrap_or_else(PoisonError::into_inner).frames.len() + } +} + +/// The reconnect wait of `shared`'s port ([`HOST_INPUT_RECONNECT_WAIT`] unless a test set one). +fn reconnect_wait(shared: &PortShared) -> Duration { + #[cfg(test)] + if let Some(wait) = shared.state.lock().unwrap_or_else(PoisonError::into_inner).reconnect_wait { + return wait; + } + let _ = shared; + HOST_INPUT_RECONNECT_WAIT } /// One frame of the port finished (written, failed or dropped): barriers waiting for it may go. @@ -393,7 +473,7 @@ fn write_with_reconnect( }; let error = first_error.take().unwrap_or(error); let deadline = *reconnect_until.get_or_insert_with(|| { - Instant::now().checked_add(HOST_INPUT_RECONNECT_WAIT).unwrap_or_else(Instant::now) + Instant::now().checked_add(reconnect_wait(shared)).unwrap_or_else(Instant::now) }); let state = TerminalHostConnectionState::from_u8( runtime.host_connection_state.load(Ordering::Acquire), diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index cfa58ffe3f62..ccab2a8d2a75 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -307,7 +307,8 @@ fn an_automation_write_gets_the_error_of_its_frame() { let resume = pty.input_port.pause_writes_for_test(); std::thread::scope(|scope| { let writing = scope.spawn(|| surface.write_bytes(b"b")); - std::thread::sleep(Duration::from_millis(100)); + // The route fails after the port took the frame (else the write is refused at enqueue, with the same error). + wait_until("the port took the write", || pty.input_port.accepted_for_test() == 1); pty.input_route.store(Some(Arc::new(input_port::InputRoute::Failed))); resume.send(()).expect("the port's writer waits"); let error = writing @@ -874,3 +875,295 @@ fn hosted_reconnect_backoff_releases_geometry_before_waiting() { "host reconnect backoff held the geometry transaction lock" ); } + +/// A host connection whose host end is closed: a send on its route fails as a lost host's send does (EPIPE). +#[cfg(unix)] +fn dead_host_route() -> input_port::InputRoute { + let (attachment, host) = crate::terminal_host_runtime::input_ack_surface_fixture(); + drop(host); + input_port::InputRoute::Hosted(attachment.input_route()) +} + +/// Waits (at most 5 s) until `ready` holds, or fails the test with `what`. +#[cfg(unix)] +fn wait_until(what: &str, ready: impl Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(5); + while !ready() { + assert!(Instant::now() < deadline, "timed out waiting until {what}"); + std::thread::sleep(Duration::from_millis(2)); + } +} + +/// The frames `host` reads until it reads nothing for `quiet` or its connection ends. +#[cfg(unix)] +fn frames_until_quiet( + host: &mut std::os::unix::net::UnixStream, + quiet: Duration, +) -> Vec { + host.set_read_timeout(Some(quiet)).unwrap(); + let mut frames = Vec::new(); + while let Ok(Some(frame)) = crate::terminal_host_protocol::read_frame( + host, + crate::terminal_host_protocol::MAX_FRAME_PAYLOAD, + ) { + frames.push(frame); + } + frames +} + +/// The payloads of the `Input` frames in `frames`, in order. +#[cfg(unix)] +fn input_payloads(frames: &[crate::terminal_host_protocol::Frame]) -> Vec> { + frames + .iter() + .filter(|frame| frame.kind == MessageKind::Input) + .map(|f| f.payload.clone()) + .collect() +} + +/// Installs a live replacement connection as the reader does (`PtyTerminalRuntime::replace_runtime` under the +/// runtime lock, which it then releases); returns the replacement's host end. +#[cfg(unix)] +fn install_replacement(pty: &PtySurface) -> std::os::unix::net::UnixStream { + let (replacement, host) = crate::terminal_host_runtime::input_ack_surface_fixture(); + let mut runtime = pty.runtime.lock().unwrap(); + pty.replace_runtime(&mut runtime, PtyRuntime::Hosted(Box::new(replacement))); + host +} + +/// R-DAEMON-STATE S1.6b review 1 (must-fix 1): after the reader installs a live replacement connection +/// (`replace_runtime`) and before its stream notify (the VT replay comes between), another writer's frame on the new +/// connection must not pass the keys the input port took while the connection was lost. The port's reconnect wait +/// ended only at the stream notify, and the barrier skipped while the port waited, so the frame went first. +#[cfg(unix)] +#[test] +fn a_frame_sent_after_a_replacement_install_waits_for_the_keys_the_port_held() { + let (_mux, surface, _host) = hosted_typing_surface("hosted-input-port-install-order", true); + let pty = surface.as_pty().expect("a terminal"); + pty.input_route.store(Some(Arc::new(dead_host_route()))); + surface.write_bytes_replying(b"k", InputReply::Accepted).expect("the port took the key"); + wait_until("the port's writer waits for a reconnect", || { + pty.input_port.reconnecting_for_test() + }); + let mut replacement_host = install_replacement(pty); + { + let runtime = pty.runtime.lock().unwrap(); + let PtyRuntime::Hosted(attachment) = &*runtime else { + panic!("the replacement is hosted"); + }; + attachment.send(MessageKind::Activate, &[]).expect("the direct frame was written"); + } + let kinds: Vec<_> = frames_until_quiet(&mut replacement_host, Duration::from_millis(1500)) + .into_iter() + .map(|frame| frame.kind) + .collect(); + let key_at = kinds.iter().position(|kind| *kind == MessageKind::Input); + let frame_at = kinds.iter().position(|kind| *kind == MessageKind::Activate); + assert!( + matches!((key_at, frame_at), (Some(key), Some(frame)) if key < frame), + "a frame sent after the replacement install passed the key the port held: {kinds:?}" + ); +} + +/// R-DAEMON-STATE S1.6b review 1 (must-fix 2): a host connection lost again later (here after a replacement that got +/// no write) gives its writes their own reconnect wait, as the old path gave each write. The port kept the first +/// loss's deadline until a write succeeded, so the later write failed at once. +#[cfg(unix)] +#[test] +fn a_later_host_loss_waits_for_its_own_reconnect() { + let (_mux, surface, _host) = hosted_typing_surface("hosted-input-port-second-loss", true); + let pty = surface.as_pty().expect("a terminal"); + pty.input_port.set_reconnect_wait_for_test(Duration::from_millis(300)); + pty.input_route.store(Some(Arc::new(dead_host_route()))); + surface.write_bytes(b"1").expect_err("a write to a lost host that never came back answered ok"); + let replacement_host = install_replacement(pty); + // The first loss's wait is long over when the replacement's host is lost too. + std::thread::sleep(Duration::from_millis(400)); + drop(replacement_host); + std::thread::scope(|scope| { + let writing = scope.spawn(|| surface.write_bytes(b"2")); + wait_until("the write waits for a reconnect or has answered", || { + writing.is_finished() || pty.input_port.reconnecting_for_test() + }); + if writing.is_finished() { + let answered = writing.join().unwrap(); + panic!("the later loss's write did not wait for its reconnect: {answered:?}"); + } + let mut next_host = install_replacement(pty); + writing.join().unwrap().expect("the write after the second reconnect"); + let written = + input_payloads(&frames_until_quiet(&mut next_host, Duration::from_millis(300))); + assert_eq!(written, vec![b"2".to_vec()], "the write reached the next replacement once"); + }); +} + +/// While the port's writer waits for a reconnect, a frame another writer sends under the runtime lock does not wait +/// for it: the connection that frame goes to is gone, and the replacement the port waits for is installed under the +/// lock that writer holds (S1.6b; a regression stalls every such frame for the order wait during a reconnect). +#[cfg(unix)] +#[test] +fn a_frame_sent_while_the_port_waits_for_a_reconnect_does_not_wait_for_it() { + let (_mux, surface, _host) = hosted_typing_surface("hosted-input-port-reconnect-skip", true); + let pty = surface.as_pty().expect("a terminal"); + pty.input_route.store(Some(Arc::new(dead_host_route()))); + surface.write_bytes_replying(b"k", InputReply::Accepted).expect("the port took the key"); + wait_until("the port's writer waits for a reconnect", || { + pty.input_port.reconnecting_for_test() + }); + let runtime = pty.runtime.lock().unwrap(); + let PtyRuntime::Hosted(attachment) = &*runtime else { + panic!("the fixture's terminal is hosted"); + }; + let started = Instant::now(); + let _ = attachment.send(MessageKind::Activate, &[]); + let took = started.elapsed(); + assert!( + took < Duration::from_millis(500), + "a frame under the runtime lock waited {took:?} for a port that waits for a reconnect" + ); +} + +/// Writes queued behind a lost host that does not come back fail together, one reconnect wait after they were sent +/// (as the old path's concurrent writes did), not one wait after another. +#[cfg(unix)] +#[test] +fn writes_queued_behind_a_lost_host_fail_together() { + let (_mux, surface, _host) = hosted_typing_surface("hosted-input-port-fail-together", true); + let pty = surface.as_pty().expect("a terminal"); + let wait = Duration::from_millis(300); + pty.input_port.set_reconnect_wait_for_test(wait); + pty.input_route.store(Some(Arc::new(dead_host_route()))); + let target: &Surface = &surface; + let started = Instant::now(); + std::thread::scope(|scope| { + let writes: Vec<_> = (0..5u8) + .map(|index| scope.spawn(move || target.write_bytes(&[b'0' + index]))) + .collect(); + for write in writes { + write + .join() + .unwrap() + .expect_err("a write to a lost host that never came back answered ok"); + } + }); + let took = started.elapsed(); + assert!( + took < wait * 3, + "5 writes behind a lost host took {took:?} with a reconnect wait of {wait:?}" + ); +} + +/// Receipted input (the `terminal.input.*` operations' path) sent after a key the input port holds reaches the host +/// after that key (S1.6b: every direct writer of the connection, not only `HostAttachment::send`). +#[cfg(unix)] +#[test] +fn receipted_input_sent_after_a_key_the_port_holds_reaches_the_host_after_it() { + let (_mux, surface, mut host) = + hosted_typing_surface("hosted-input-port-receipted-order", true); + let pty = surface.as_pty().expect("a terminal"); + let resume = pty.input_port.pause_writes_for_test(); + surface.write_bytes_replying(b"k", InputReply::Accepted).expect("the port took the key"); + std::thread::scope(|scope| { + let receipted = scope.spawn(|| { + let runtime = pty.runtime.lock().unwrap(); + let PtyRuntime::Hosted(attachment) = &*runtime else { + panic!("the fixture's terminal is hosted"); + }; + attachment.begin_input_confirmed(b"r").map(|_| ()).map_err(|_| "not written") + }); + std::thread::sleep(Duration::from_millis(200)); + resume.send(()).expect("the port's writer waits"); + receipted.join().unwrap().expect("the receipted input was written"); + }); + drop(resume); + let written = input_payloads(&frames_until_quiet(&mut host, Duration::from_millis(300))); + assert_eq!( + written, + vec![b"k".to_vec(), b"r".to_vec()], + "receipted input passed the key the port held" + ); +} + +/// R-DAEMON-STATE S1.6b review 1 (item 6): a write whose wait for the port ends before the port's writer took its +/// frame is never written: its error says so, and nothing reaches the host later (the creating-input flush gives +/// such bytes back, so writing them too would type them twice). +#[cfg(unix)] +#[test] +fn a_write_whose_wait_ends_before_the_port_takes_it_is_never_written() { + let (_mux, surface, mut host) = hosted_typing_surface("hosted-input-port-reply-timeout", true); + let pty = surface.as_pty().expect("a terminal"); + pty.input_port.set_reply_wait_for_test(Duration::from_millis(300)); + let resume = pty.input_port.pause_writes_for_test(); + // The writer holds a key at its pause; the write queues behind it. + surface.write_bytes_replying(b"k", InputReply::Accepted).expect("the port took the key"); + wait_until("the writer took the key", || pty.input_port.queued_frames_for_test() == 0); + let error = + surface.write_bytes(b"x").expect_err("a write the port did not reach in time answered ok"); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut, "{error}"); + assert!( + !input_port::input_may_be_written(&error), + "a frame the port never took may be written: {error}" + ); + drop(resume); + let written = input_payloads(&frames_until_quiet(&mut host, Duration::from_millis(500))); + assert_eq!( + written, + vec![b"k".to_vec()], + "a write that answered 'not written' reached the host" + ); +} + +/// R-DAEMON-STATE S1.6b review 1 (item 6): a write whose wait ends while the port's writer is writing its frame says +/// that the bytes may still be written, so a caller that gives input back keeps these. +#[cfg(unix)] +#[test] +fn a_write_whose_wait_ends_during_its_write_may_be_written() { + let (_mux, surface, mut host) = + hosted_typing_surface("hosted-input-port-reply-in-flight", true); + let pty = surface.as_pty().expect("a terminal"); + pty.input_port.set_reply_wait_for_test(Duration::from_millis(300)); + let resume = pty.input_port.pause_writes_for_test(); + let error = surface.write_bytes(b"y").expect_err("a write still being written answered ok"); + assert!( + input_port::input_may_be_written(&error), + "a frame the port was writing was 'not written': {error}" + ); + drop(resume); + let written = input_payloads(&frames_until_quiet(&mut host, Duration::from_millis(500))); + assert_eq!(written, vec![b"y".to_vec()], "the frame the port was writing reached the host"); +} + +/// R-DAEMON-STATE S1.6b review 1 (item 6): the creating-input flush gives back the bytes it could not write, but not +/// bytes whose write may still reach the program; giving those back too types them twice. +#[cfg(unix)] +#[test] +fn the_creating_input_flush_keeps_input_that_may_have_been_written() { + let (mux, surface, mut host) = hosted_typing_surface("hosted-input-port-creating-flush", true); + let pty = surface.as_pty().expect("a terminal"); + let terminal_id = surface.terminal_host_identity().expect("a hosted terminal").terminal_id; + let generation = + mux.begin_creating_input(&terminal_id).expect("no live terminal has the id yet"); + let given_back = Arc::new(Mutex::new(Vec::new())); + for bytes in [&b"a"[..], &b"b"[..]] { + let given_back = given_back.clone(); + let notice: crate::mux::creating_input::UnclaimedNotice = + Arc::new(move |_terminal: &str, bytes: &[u8], _reason: &str| { + given_back.lock().unwrap().extend_from_slice(bytes); + }); + mux.write_terminal_input(&terminal_id, 7, bytes, false, notice) + .expect("queued while creating"); + } + mux.insert_surface_runtime_for_test(surface.clone()); + pty.input_port.set_reply_wait_for_test(Duration::from_millis(300)); + let resume = pty.input_port.pause_writes_for_test(); + // The flush writes "a"; the port's writer holds it at its pause until the flush's wait ends. + mux.finish_creating_input(&terminal_id, generation, None); + assert_eq!( + given_back.lock().unwrap().as_slice(), + b"b", + "the flush gave back input whose write may still reach the program" + ); + drop(resume); + let written = input_payloads(&frames_until_quiet(&mut host, Duration::from_millis(500))); + assert_eq!(written, vec![b"a".to_vec()], "the input the flush kept reached the host once"); +} From bbc44c24ae40cdc9567c45b79494687f064c6d44 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:17:57 +0000 Subject: [PATCH 39/42] fix(cmux-tui-core): a route publication ends the input port's reconnect wait; each frame waits for a reconnect from when the port took it; a timed-out write is cancelled or marked possibly written (#35082 review 1) - Must-fix 1: replace_runtime and input_route_failed call InputPort::route_changed. A port-owned route epoch ends the writer's reconnect wait and clears reconnecting under the port lock, so after the reader installs a live replacement the port writes the keys it holds to it before any other frame (before, the writer slept on the stream revision until the reader's notify after the VT replay, and the barrier skipped meanwhile). - Must-fix 2: each frame waits for a reconnect until its own deadline, HOST_INPUT_RECONNECT_WAIT after the port took it, as the old path's write waited from its call. The queue-wide reconnect_until outlived its loss, so a later loss's writes failed at once. Writes queued behind a host that does not come back still fail together. - Item 6: InputPort::write cancels a frame its waiter gave up on before the writer took it (the error says not written, and it never is); a wait that ends during the write answers InputMayBeWritten, and the creating-input flush does not give those bytes back (that typed them twice). - Item 5: the order barrier ends by the caller's own deadline (control requests, cell pixels). - Item 3: the order-wait comment states the bound; the clipboard replier uses the same deadline. - Item 8: the port and input docs describe the current rules. Tests (red at 2133c10ae2f): a_frame_sent_after_a_replacement_install_waits_for_the_keys_the_port_held, a_later_host_loss_waits_for_its_own_reconnect, a_write_whose_wait_ends_before_the_port_takes_it_is_never_written, a_write_whose_wait_ends_during_its_write_may_be_written, the_creating_input_flush_keeps_input_that_may_have_been_written. --- .../cmux-tui-core/src/mux/creating_input.rs | 10 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 2 +- .../crates/cmux-tui-core/src/surface/input.rs | 27 +-- .../cmux-tui-core/src/surface/input_port.rs | 218 +++++++++++++----- .../src/surface/terminal_runtime.rs | 5 + .../shared/attachment.rs | 36 ++- .../shared/attachment/pending_control.rs | 2 +- .../shared/clipboard_read.rs | 4 +- 8 files changed, 213 insertions(+), 91 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs index cf18ac8ac31c..b69c9ff6bcbe 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/creating_input.rs @@ -289,7 +289,8 @@ impl Mux { }; if let Err(error) = written { // The terminal took the earlier bytes; the rest, and all queued since, go - // back to their senders. + // back to their senders. Bytes whose write may still reach the program (its + // wait ended during the write) are not given back: that would type them twice. eprintln!( "cmux-tui: queued input for terminal {terminal_id} not written: {error}" ); @@ -300,7 +301,12 @@ impl Mux { inputs.ended.notify_all(); creating.map(|creating| creating.queued).unwrap_or_default() }; - let mut rest: Vec<_> = batch.into_iter().skip(index).collect(); + let first_given_back = if crate::surface::input_may_be_written(&error) { + index.saturating_add(1) + } else { + index + }; + let mut rest: Vec<_> = batch.into_iter().skip(first_given_back).collect(); rest.extend(later); return give_back(terminal_id, rest, "write-failed"); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 31485adbe130..48e7c2e0e640 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -66,7 +66,7 @@ mod hosted_stager; pub(crate) mod input_port; #[cfg(any(unix, windows))] use hosted_stager::{HostedFrameStager, HostedTransition}; -pub(crate) use input_port::InputReply; +pub(crate) use input_port::{InputReply, input_may_be_written}; #[cfg(test)] use options::child_term_for; use options::configure_agent_browser_session; diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs index a68338ed09ca..f5601e7fb7b9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input.rs @@ -9,7 +9,9 @@ use super::*; /// The wait blocks its caller: one control connection's next request, or /// the TUI's input worker. It applies only while a host connection is lost, /// and a replacement host usually installs within a second. On the input lane -/// the terminal's input port waits instead (`surface::input_port`). +/// the terminal's input port waits on its own thread, for each frame from the +/// moment the port took it (`surface::input_port`); the person's keys do not +/// wait, and every other write waits for its frame's result. pub(super) const HOST_INPUT_RECONNECT_WAIT: Duration = Duration::from_secs(5); /// Whether a host send failed because the host connection is gone (the @@ -73,7 +75,8 @@ impl Surface { } /// On the input lane (R-DAEMON-STATE stage S1 step 6): hands interactive input for a hosted terminal to the - /// terminal's input port and returns once the port took it; `None` when the lane is off or the terminal is not + /// terminal's input port. The person's keys ([`InputReply::Accepted`]) return once the port took them; every other + /// write returns with its frame's result (`InputPort::write`). `None` when the lane is off or the terminal is not /// hosted (the caller writes as before). The final screen of an exited host drops the input, as before. #[cfg(any(unix, windows))] fn enqueue_on_input_port( @@ -103,25 +106,9 @@ impl Surface { } Some(input_port::InputRoute::Hosted(_)) => Some(match reply { InputReply::Accepted => { - pty.input_port.enqueue(&pty.terminal, kind, bytes, None).map_err(Into::into) - } - InputReply::Written => { - let (done, written) = std::sync::mpsc::sync_channel(1); - pty.input_port - .enqueue(&pty.terminal, kind, bytes, Some(done)) - .map_err(std::io::Error::from) - .and_then(|()| match written.recv_timeout(pty.input_port.reply_wait()) { - Ok(result) => result, - Err(RecvTimeoutError::Timeout) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "terminal input still waits in the terminal's input port", - )), - Err(RecvTimeoutError::Disconnected) => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "the terminal's input port closed before it wrote the input", - )), - }) + pty.input_port.enqueue(&pty.terminal, kind, bytes).map_err(Into::into) } + InputReply::Written => pty.input_port.write(&pty.terminal, kind, bytes), }), } } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index 04907060eaa7..a37ca8d5bed8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -13,22 +13,30 @@ //! //! - The writer writes on the route of the terminal's current host connection, which the terminal publishes each //! time the connection changes (`PtyTerminalRuntime::replace_runtime`), so it never takes the runtime lock either. -//! - A lost host connection is retried as `write_bytes` did (cx-6so.49): the writer waits for the reader's next -//! stream change and writes to the new host, until `HOST_INPUT_RECONNECT_WAIT`. +//! - A lost host connection is retried as `write_bytes` did (cx-6so.49): the writer waits until the terminal +//! publishes another route ([`InputPort::route_changed`]) and writes to it, until the frame's own deadline, +//! `HOST_INPUT_RECONNECT_WAIT` after the port took it. The old path's deadline started at the write call, so writes +//! queued behind a host that does not come back fail together, and a later loss gives its writes their own wait. //! - A full port refuses at once with a typed error: nothing is dropped silently and nothing waits. Each queued //! frame counts its bookkeeping too ([`PORT_FRAME_COST`]), so many tiny frames cannot grow the queue without bound. //! - A connection the reader shut down (`Failed`, it keeps retrying: [`InputRoute::Failed`]) or a reader that ended //! takes no input: the write is refused at once, as the old path's write failed then //! (`Surface::enqueue_on_input_port`). A replacement's route is accepted as soon as it is installed. While a lost //! connection is still `Reconnecting`, input is accepted and the writer waits for the reconnect. -//! - Input the port took and then could not write (the host is gone and no new one came) is lost, as a failed write -//! was before; the request had already been answered, so the port logs it. -//! - Stage S1 step 6b, before the default flip: every other frame of the connection (viewer size, cell pixels, -//! receipted input, clear history, Kitty limits, clipboard replies, terminate) still writes directly, so while the -//! port is behind such a frame can go before a key the port still holds. +//! - Only the person's keys answer when the port took them ([`InputReply::Accepted`]); a key the port then could not +//! write (the host is gone and no new one came) is lost, as a failed write was before, and the port logs it. Every +//! other write waits for its frame's result ([`InputPort::write`]), at most [`INPUT_WRITE_REPLY_WAIT`]: a wait that +//! ends before the writer took the frame cancels it, and one that ends during its write answers +//! [`InputMayBeWritten`]. +//! - Every other frame of the connection (viewer size, cell pixels, receipted input, clear history, Kitty limits, +//! clipboard replies, terminate) first waits for the frames the port took before it ([`PortOrder::barrier`], at most +//! 2 s or its caller's sooner deadline). Order holds while the port drains within that bound: a frame still in its +//! write after it (a large paste to a host that reads slowly) lets the waiting frame go. The barrier does not wait +//! while the writer waits for a reconnect: the connection that frame goes to is gone, and the replacement is +//! installed under the runtime lock that frame's writer may hold. The install ends the writer's wait at once. use std::collections::VecDeque; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Condvar, Mutex, PoisonError, Weak}; use std::time::{Duration, Instant}; @@ -109,7 +117,28 @@ impl From for std::io::Error { } /// Where the port reports one frame's write to a writer that waits for it ([`InputReply::Written`]). -pub(crate) type FrameDone = std::sync::mpsc::SyncSender>; +type FrameDone = std::sync::mpsc::SyncSender>; + +/// A waited-for frame is still queued, taken by the writer to write, or cancelled by its waiter. +const FRAME_QUEUED: u8 = 0; +const FRAME_TAKEN: u8 = 1; +const FRAME_CANCELLED: u8 = 2; + +/// The writer waiting for one frame's result ([`InputPort::write`]). +struct FrameWaiter { + done: FrameDone, + /// [`FRAME_QUEUED`], then [`FRAME_TAKEN`] (the port's writer) or [`FRAME_CANCELLED`] (the waiter gave up first). + state: Arc, +} + +impl FrameWaiter { + /// The port's writer takes the frame to write it; false when its waiter already gave up on it. + fn take(&self) -> bool { + self.state + .compare_exchange(FRAME_QUEUED, FRAME_TAKEN, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} /// What a write to a hosted terminal's input port answers (S1.6b): the person's keys return when the port took them; /// every other writer (a script's `send`, a resource input effect, the creating-input flush, an image paste) waits @@ -125,8 +154,11 @@ pub(crate) enum InputReply { struct PortFrame { kind: MessageKind, bytes: Vec, + /// The end of this frame's wait for a lost connection to come back: `HOST_INPUT_RECONNECT_WAIT` after the port + /// took it, as the old path's write waited from its call. + reconnect_by: Instant, /// The waiting writer of an [`InputReply::Written`] write; dropped unanswered when the frame is dropped. - done: Option, + waiter: Option, } impl PortFrame { @@ -149,6 +181,8 @@ struct PortState { /// The writer waits for a reconnect: a barrier does not wait then; the connection it would write to is gone, and /// the install it waits for may need the runtime lock that barrier's caller holds. reconnecting: bool, + /// Moves each time the terminal publishes a route ([`InputPort::route_changed`]): ends the writer's reconnect wait. + route_epoch: u64, /// Tests: the writer waits here before it writes each frame. #[cfg(test)] paused: Option>, @@ -167,6 +201,12 @@ struct PortShared { finished_changed: Condvar, } +impl PortShared { + fn route_epoch(&self) -> u64 { + self.state.lock().unwrap_or_else(PoisonError::into_inner).route_epoch + } +} + /// The order of one terminal's input port for the other writers of its host connection (DESIGN 2.6.1, S1.6b): a /// viewer size, a cell-pixel size, receipted input, a clear history, Kitty limits, a clipboard reply or a terminate /// issued after a key the port took reaches the host after that key. @@ -210,20 +250,84 @@ impl Drop for InputPort { } impl InputPort { - /// Takes `bytes` for `terminal`'s current host connection and returns at once; the terminal's writer thread - /// writes them after every earlier write of the port. + /// Takes `bytes` for `terminal`'s current host connection and returns at once ([`InputReply::Accepted`]); the + /// terminal's writer thread writes them after every earlier write of the port. pub(crate) fn enqueue( &self, terminal: &Arc, kind: MessageKind, bytes: &[u8], - done: Option, + ) -> Result<(), InputRefused> { + self.enqueue_frame(terminal, kind, bytes, None) + } + + /// [`Self::enqueue`] for a writer that waits for its frame's result ([`InputReply::Written`]), as the old path's + /// caller waited for its write, at most [`Self::reply_wait`]. A wait that ends before the port's writer took the + /// frame cancels it: the error says not written, and it never is. One that ends during its write answers the + /// write's result if it ended meanwhile, else [`InputMayBeWritten`]. + pub(crate) fn write( + &self, + terminal: &Arc, + kind: MessageKind, + bytes: &[u8], + ) -> std::io::Result<()> { + let (done, written) = std::sync::mpsc::sync_channel(1); + let state = Arc::new(AtomicU8::new(FRAME_QUEUED)); + let waiter = FrameWaiter { done, state: Arc::clone(&state) }; + self.enqueue_frame(terminal, kind, bytes, Some(waiter))?; + match written.recv_timeout(self.reply_wait()) { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "the terminal's input port stopped before it wrote the input", + )), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + if state + .compare_exchange( + FRAME_QUEUED, + FRAME_CANCELLED, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal input not written: the terminal's input port did not reach it in time", + )); + } + written.try_recv().unwrap_or_else(|_| { + Err(std::io::Error::new(std::io::ErrorKind::TimedOut, InputMayBeWritten)) + }) + } + } + } + + /// The terminal published a new input route (a replacement connection, a failed link, an exited host; the caller + /// holds the runtime lock): a writer waiting for a reconnect tries the new route at once, and a frame of another + /// writer waits for the port again, whose next write goes to the new connection ahead of it. + pub(crate) fn route_changed(&self) { + let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); + state.route_epoch = state.route_epoch.wrapping_add(1); + state.reconnecting = false; + drop(state); + self.shared.changed.notify_all(); + } + + fn enqueue_frame( + &self, + terminal: &Arc, + kind: MessageKind, + bytes: &[u8], + waiter: Option, ) -> Result<(), InputRefused> { if bytes.len() > MAX_FRAME_PAYLOAD { return Err(InputRefused::TooLarge); } + let reconnect_by = + Instant::now().checked_add(reconnect_wait(&self.shared)).unwrap_or_else(Instant::now); // Copied before the lock, so another write to this terminal never waits for this one's copy. - let frame = PortFrame { kind, bytes: bytes.to_vec(), done }; + let frame = PortFrame { kind, bytes: bytes.to_vec(), reconnect_by, waiter }; let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); let Some(total) = state .queued_bytes @@ -377,10 +481,13 @@ impl Drop for WriterUnwind<'_> { fn run_writer(shared: &PortShared, terminal: &Weak) { let _unwind = WriterUnwind(shared); - // One reconnect wait for the queue: set by the first frame that finds the connection lost, cleared by the next - // write that succeeds, so frames queued behind a host that never comes back fail together, not 5 s each. - let mut reconnect_until = None; while let Some(frame) = next_frame(shared) { + // Its waiter gave up before the writer reached it (`InputPort::write`): it answered "not written", so the + // frame is not written. + if frame.waiter.as_ref().is_some_and(|waiter| !waiter.take()) { + finish_frames(shared, 1); + continue; + } #[cfg(test)] { let paused = shared.state.lock().unwrap_or_else(PoisonError::into_inner).paused.take(); @@ -394,13 +501,12 @@ fn run_writer(shared: &PortShared, terminal: &Weak) { finish_frames(shared, 1); return; }; - let written = - write_with_reconnect(shared, &runtime, frame.kind, &frame.bytes, &mut reconnect_until); + let written = write_with_reconnect(shared, &runtime, &frame); finish_frames(shared, 1); - match (written, frame.done) { + match (written, frame.waiter) { // The waiting writer answers its request with the result. - (written, Some(done)) => { - let _ = done.send(written); + (written, Some(waiter)) => { + let _ = waiter.done.send(written); } (Err(error), None) => eprintln!( "cmux-tui: terminal input not written ({} bytes, already accepted): {error}", @@ -411,39 +517,53 @@ fn run_writer(shared: &PortShared, terminal: &Weak) { } } -/// Marks the writer waiting for a reconnect while it lives, so barriers do not wait for it meanwhile. -struct Reconnecting<'a>(&'a PortShared); - -impl<'a> Reconnecting<'a> { - fn start(shared: &'a PortShared) -> Self { - shared.state.lock().unwrap_or_else(PoisonError::into_inner).reconnecting = true; - shared.finished_changed.notify_all(); - Self(shared) +/// Waits until the terminal publishes a route after the one seen at epoch `observed` ([`InputPort::route_changed`]), +/// until the port closes or until `deadline`; true when a new route came. Marks the writer waiting for a reconnect +/// meanwhile, so barriers do not wait for it (a new route clears the mark at once). +fn wait_for_route(shared: &PortShared, observed: u64, deadline: Instant) -> bool { + let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); + if state.route_epoch != observed { + return true; } -} - -impl Drop for Reconnecting<'_> { - fn drop(&mut self) { - self.0.state.lock().unwrap_or_else(PoisonError::into_inner).reconnecting = false; + if state.closed || Instant::now() >= deadline { + return false; } + state.reconnecting = true; + shared.finished_changed.notify_all(); + let published = loop { + if state.route_epoch != observed { + break true; + } + let now = Instant::now(); + if state.closed || now >= deadline { + break false; + } + state = shared + .changed + .wait_timeout(state, deadline.saturating_duration_since(now)) + .unwrap_or_else(PoisonError::into_inner) + .0; + }; + state.reconnecting = false; + published } /// One frame on the terminal's current host connection, across a lost connection: the rule of -/// `Surface::send_hosted_input`, on the port's thread, reading the published route instead of the runtime lock. +/// `Surface::send_hosted_input`, on the port's thread, reading the published route instead of the runtime lock. The +/// frame waits for a reconnect until its own deadline (`PortFrame::reconnect_by`). fn write_with_reconnect( shared: &PortShared, runtime: &PtyTerminalRuntime, - kind: MessageKind, - bytes: &[u8], - reconnect_until: &mut Option, + frame: &PortFrame, ) -> std::io::Result<()> { let mut first_error = None; loop { - // Read the revision before the send: an install after it wakes the wait below, so no reconnect is missed. - let observed = runtime.stream_progress.revision(); + // Read the route epoch before the route: a route published after it ends the wait below, so no reconnect is + // missed (`PtyTerminalRuntime::replace_runtime` stores the route, then moves the epoch). + let observed = shared.route_epoch(); let route = runtime.input_route.load_full(); let sent = match route.as_deref() { - Some(InputRoute::Hosted(route)) => route.send(kind, bytes), + Some(InputRoute::Hosted(route)) => route.send(frame.kind, &frame.bytes), // Keep-on-exit: the final screen drops input; input that already failed keeps its error. Some(InputRoute::Exited) => return first_error.map_or(Ok(()), Err), Some(InputRoute::Failed) => { @@ -464,17 +584,11 @@ fn write_with_reconnect( } }; let error = match sent { - Ok(()) => { - *reconnect_until = None; - return Ok(()); - } + Ok(()) => return Ok(()), Err(error) if super::input::host_connection_lost(&error) => error, Err(error) => return Err(error), }; let error = first_error.take().unwrap_or(error); - let deadline = *reconnect_until.get_or_insert_with(|| { - Instant::now().checked_add(reconnect_wait(shared)).unwrap_or_else(Instant::now) - }); let state = TerminalHostConnectionState::from_u8( runtime.host_connection_state.load(Ordering::Acquire), ); @@ -483,11 +597,7 @@ fn write_with_reconnect( TerminalHostConnectionState::Failed | TerminalHostConnectionState::Exited ) || runtime.reader_completion.is_complete() || runtime.owner_detaching.load(Ordering::Acquire) - || Instant::now() >= deadline - || { - let _reconnecting = Reconnecting::start(shared); - runtime.stream_progress.wait_for_change(observed, deadline).is_none() - } + || !wait_for_route(shared, observed, frame.reconnect_by) { return Err(error); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index dc2cecf12bae..7132803f976d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -74,6 +74,9 @@ impl PtyTerminalRuntime { host.set_input_order(self.input_port.order()); } self.input_route.store(Self::input_route_of(runtime)); + // After the store: a port writer waiting for a reconnect writes the keys it holds to the new connection now, + // before any other frame on it (#35082 review 1). + self.input_port.route_changed(); } /// The reader shut the current host connection down and keeps retrying (`Failed`): input is refused until a @@ -82,6 +85,8 @@ impl PtyTerminalRuntime { #[cfg(any(unix, windows))] pub(super) fn input_route_failed(&self, _runtime: &PtyRuntime) { self.input_route.store(Some(Arc::new(input_port::InputRoute::Failed))); + // A port writer waiting for a reconnect fails its frame now, as the old path's write failed then. + self.input_port.route_changed(); } } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs index 7ec287e730c3..841be35cf8c4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment.rs @@ -13,10 +13,18 @@ use std::time::{Duration, Instant}; use anyhow::Context; -/// The longest a frame other than the input port's own waits for the port's earlier keys (S1.6b): as long as one -/// frame write may block on a host socket. +/// The longest a frame other than the input port's own waits for the frames the port took before it (S1.6b). Order +/// holds while the port drains within it: a frame still in its write after it (a large paste to a host that reads +/// slowly) lets the waiting frame go, and the frames the port queued behind that write may then follow it. const INPUT_ORDER_WAIT: Duration = Duration::from_secs(2); +/// The end of a wait for the input port's earlier frames: [`INPUT_ORDER_WAIT`] from now, or the caller's own +/// `deadline` when it is sooner. +pub(super) fn input_order_deadline(deadline: Option) -> Instant { + let cap = Instant::now().checked_add(INPUT_ORDER_WAIT).unwrap_or_else(Instant::now); + deadline.map_or(cap, |deadline| deadline.min(cap)) +} + pub(crate) use super::super::sys::SpawnedHostProcess; use super::super::sys::{HostStream, PtyCustody, connect_with_retry}; use super::super::*; @@ -196,7 +204,7 @@ impl HostAttachment { } pub fn send(&self, kind: MessageKind, payload: &[u8]) -> std::io::Result<()> { - self.wait_input_order(); + self.wait_input_order(None); send_host_frame(&self.writer, self.protocol_version, kind, payload) } @@ -205,17 +213,25 @@ impl HostAttachment { self.input_order = Some(order); } - /// Before a frame other than the port's own: the keys the port took before are written first (S1.6b). - pub(crate) fn wait_input_order(&self) { + /// Before a frame other than the port's own: the frames the port took before are written first (S1.6b), within + /// [`input_order_deadline`] of the caller's `deadline`. + pub(crate) fn wait_input_order(&self, deadline: Option) { if let Some(order) = &self.input_order { - order - .barrier(Instant::now().checked_add(INPUT_ORDER_WAIT).unwrap_or_else(Instant::now)); + order.barrier(input_order_deadline(deadline)); } } - /// The writer for a frame other than the port's own, after the port's earlier keys (S1.6b). + /// The writer for a frame other than the port's own, after the port's earlier frames (S1.6b). fn direct_writer(&self) -> std::sync::MutexGuard<'_, HostStream> { - self.wait_input_order(); + self.direct_writer_until(None) + } + + /// [`Self::direct_writer`] for a caller with its own deadline: the wait for the port ends by it. + fn direct_writer_until( + &self, + deadline: Option, + ) -> std::sync::MutexGuard<'_, HostStream> { + self.wait_input_order(deadline); self.writer.lock().recover_poison() } @@ -428,7 +444,7 @@ impl HostAttachment { frame.version = self.protocol_version; frame.request_id = request_id; let write_result = { - let mut writer = self.direct_writer(); + let mut writer = self.direct_writer_until(Some(deadline)); write_frame(&mut *writer, &frame).map_err(protocol_io_error) }; if let Err(error) = write_result { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs index 64f4e6931fe1..b0ae4b1bdcf3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/attachment/pending_control.rs @@ -53,7 +53,7 @@ impl HostAttachment { frame.version = self.protocol_version; frame.request_id = request_id; let write_result = { - let mut writer = self.direct_writer(); + let mut writer = self.direct_writer_until(Some(deadline)); let result = write_frame(&mut *writer, &frame).map_err(protocol_io_error); if result.is_err() { let _ = writer.shutdown(std::net::Shutdown::Both); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs index ea8cb46241f0..0098cd2a2372 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime/shared/clipboard_read.rs @@ -655,9 +655,7 @@ impl ClipboardReplier { } let reply = encode_clipboard_read_reply(token, text); if let Some(order) = &self.input_order { - order.barrier( - Instant::now().checked_add(Duration::from_secs(2)).unwrap_or_else(Instant::now), - ); + order.barrier(super::attachment::input_order_deadline(None)); } send_host_frame(&writer, self.protocol_version, MessageKind::ClipboardReadReply, &reply)?; Ok(true) From f9c75f1f7a7b123c62d0253ea81be5ea6a2c0cc8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:33:13 +0000 Subject: [PATCH 40/42] test(cmux-tui-core): the review-1 port tests name Frame and input_may_be_written through the module scope (clippy unused_qualifications) lint (linux) at bbc44c24ae4 (runs 41806402684, 41806407494) failed clippy -D warnings on four qualified paths in surface/tests/hosted.rs; both names are in scope through use super::*. --- .../cmux-tui-core/src/surface/tests/hosted.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index ccab2a8d2a75..f8985c623f29 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -896,10 +896,7 @@ fn wait_until(what: &str, ready: impl Fn() -> bool) { /// The frames `host` reads until it reads nothing for `quiet` or its connection ends. #[cfg(unix)] -fn frames_until_quiet( - host: &mut std::os::unix::net::UnixStream, - quiet: Duration, -) -> Vec { +fn frames_until_quiet(host: &mut std::os::unix::net::UnixStream, quiet: Duration) -> Vec { host.set_read_timeout(Some(quiet)).unwrap(); let mut frames = Vec::new(); while let Ok(Some(frame)) = crate::terminal_host_protocol::read_frame( @@ -913,7 +910,7 @@ fn frames_until_quiet( /// The payloads of the `Input` frames in `frames`, in order. #[cfg(unix)] -fn input_payloads(frames: &[crate::terminal_host_protocol::Frame]) -> Vec> { +fn input_payloads(frames: &[Frame]) -> Vec> { frames .iter() .filter(|frame| frame.kind == MessageKind::Input) @@ -1100,10 +1097,7 @@ fn a_write_whose_wait_ends_before_the_port_takes_it_is_never_written() { let error = surface.write_bytes(b"x").expect_err("a write the port did not reach in time answered ok"); assert_eq!(error.kind(), std::io::ErrorKind::TimedOut, "{error}"); - assert!( - !input_port::input_may_be_written(&error), - "a frame the port never took may be written: {error}" - ); + assert!(!input_may_be_written(&error), "a frame the port never took may be written: {error}"); drop(resume); let written = input_payloads(&frames_until_quiet(&mut host, Duration::from_millis(500))); assert_eq!( @@ -1125,7 +1119,7 @@ fn a_write_whose_wait_ends_during_its_write_may_be_written() { let resume = pty.input_port.pause_writes_for_test(); let error = surface.write_bytes(b"y").expect_err("a write still being written answered ok"); assert!( - input_port::input_may_be_written(&error), + input_may_be_written(&error), "a frame the port was writing was 'not written': {error}" ); drop(resume); From b6b0b37f3b08f5504fad0f21bfdee3341b9ae8df Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:30:12 +0000 Subject: [PATCH 41/42] test(cmux-tui-core): a write waiting for a reconnect fails when the reader gives up (R-DAEMON-STATE S1.6c red, #35082 review 2) The reader's end, give-up and failed-reconnect paths now call PtyTerminalRuntime::reader_gave_up, which still only notifies stream_progress (today's behavior). The port's writer waits on its route epoch since #35082 review 1, so the wake does not reach it: a_write_waiting_for_a_reconnect_fails_when_the_reader_gives_up waits its full reconnect deadline (5 s) instead of failing at once. --- .../src/surface/hosted_reader.rs | 2 +- .../src/surface/hosted_reader/reconnect.rs | 2 +- .../src/surface/reconnect_backoff.rs | 2 +- .../src/surface/terminal_runtime.rs | 6 ++++ .../cmux-tui-core/src/surface/tests/hosted.rs | 30 +++++++++++++++++++ 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader.rs b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader.rs index 4395de351f04..12f82f5ced6c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader.rs @@ -326,7 +326,7 @@ struct WakeInputOnReaderEnd(Arc); impl Drop for WakeInputOnReaderEnd { fn drop(&mut self) { if let Some(pty) = self.0.as_pty() { - pty.stream_progress.notify(); + pty.reader_gave_up(); // A reader that gave up on its host leaves the terminal `failed`: // push the tab here, with no lock held, so clients show it. if matches!( diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs index 01b698917d51..37b24d8495f6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/hosted_reader/reconnect.rs @@ -299,7 +299,7 @@ impl HostedReader { if !reconnect_mux.terminal_host_connection_lost(surface.id, &identity) { pty.host_connection_state .store(TerminalHostConnectionState::Failed as u8, Ordering::Release); - pty.stream_progress.notify(); + pty.reader_gave_up(); return None; } // A resync's reconnect that failed is a loss clients have not heard of. diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs b/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs index 10c0577a4dbc..7ebcaec46cfc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/reconnect_backoff.rs @@ -85,7 +85,7 @@ impl TerminalHostReconnectBackoff { host.disconnect(); } // Input waiting out the reconnect (surface/input.rs) fails now. - pty.stream_progress.notify(); + pty.reader_gave_up(); return false; }; #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index 7132803f976d..a3ff8149a633 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -79,6 +79,12 @@ impl PtyTerminalRuntime { self.input_port.route_changed(); } + /// The reader will install no host connection now (it ended, gave the host up, or the owner detaches the + /// terminal): input that waits out a lost connection fails now, as the old path's write failed then. + pub(super) fn reader_gave_up(&self) { + self.stream_progress.notify(); + } + /// The reader shut the current host connection down and keeps retrying (`Failed`): input is refused until a /// replacement connection is installed ([`Self::replace_runtime`] publishes its route), as the old path's write /// to the shut connection failed. Takes the held runtime so it is ordered with every replacement. diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs index f8985c623f29..027f5c24b33f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/tests/hosted.rs @@ -1161,3 +1161,33 @@ fn the_creating_input_flush_keeps_input_that_may_have_been_written() { let written = input_payloads(&frames_until_quiet(&mut host, Duration::from_millis(500))); assert_eq!(written, vec![b"a".to_vec()], "the input the flush kept reached the host once"); } + +/// R-DAEMON-STATE S1.6c (#35082 review 2, should-do 1): a write that waits out a lost host connection fails as soon +/// as the reader gives that host up (the link is `Failed` and nothing will install a replacement), as the old path's +/// write failed then, not at its own reconnect deadline. The reader's wake reached only `stream_progress`, which the +/// port's writer no longer waits on. +#[cfg(unix)] +#[test] +fn a_write_waiting_for_a_reconnect_fails_when_the_reader_gives_up() { + let (_mux, surface, _host) = hosted_typing_surface("hosted-input-port-give-up", true); + let pty = surface.as_pty().expect("a terminal"); + pty.input_port.set_reconnect_wait_for_test(Duration::from_secs(5)); + pty.input_route.store(Some(Arc::new(dead_host_route()))); + std::thread::scope(|scope| { + let writing = scope.spawn(|| { + let started = Instant::now(); + (surface.write_bytes(b"g"), started.elapsed()) + }); + wait_until("the write waits for a reconnect", || pty.input_port.reconnecting_for_test()); + // The reader gives the host up, as `wait_or_fail` does at the end of its budget. + pty.host_connection_state + .store(TerminalHostConnectionState::Failed as u8, Ordering::Release); + pty.reader_gave_up(); + let (written, took) = writing.join().unwrap(); + written.expect_err("a write to a host the reader gave up on answered ok"); + assert!( + took < Duration::from_secs(2), + "a write to a host the reader gave up on waited {took:?}, its reconnect deadline, not the give-up" + ); + }); +} From 860edd9053e52d86a46182b7d6b6c73410761b3d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:33:02 +0000 Subject: [PATCH 42/42] fix(cmux-tui-core): the reader's end, give-up and the owner's detach end the input port's reconnect wait (R-DAEMON-STATE S1.6c, #35082 review 2) Since review 1 the port's writer waits on its own route epoch, so the reader's stream_progress wake no longer reached it: a Written write behind a host the reader gave up on failed only at its reconnect deadline (5 s), not at once as the old path's write did. InputPort::reconnect_ended moves the epoch with no new route; the writer checks the route and the terminal's state again and fails its frame (Failed state, ended reader, or owner detaching). - PtyTerminalRuntime::reader_gave_up (WakeInputOnReaderEnd, wait_or_fail's give-up, a failed reconnect completion) notifies the stream and ends the port's wait. - The four owner_detaching stores in shutdown.rs end the port's wait (only the port lock, under the runtime lock). Test: a_write_waiting_for_a_reconnect_fails_when_the_reader_gives_up (red at b6b0b37f3b0). --- .../cmux-tui-core/src/surface/input_port.rs | 19 ++++++++++++++++--- .../cmux-tui-core/src/surface/shutdown.rs | 12 ++++++++++++ .../src/surface/terminal_runtime.rs | 2 ++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs index a37ca8d5bed8..fdab5a3fd50d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/input_port.rs @@ -181,7 +181,8 @@ struct PortState { /// The writer waits for a reconnect: a barrier does not wait then; the connection it would write to is gone, and /// the install it waits for may need the runtime lock that barrier's caller holds. reconnecting: bool, - /// Moves each time the terminal publishes a route ([`InputPort::route_changed`]): ends the writer's reconnect wait. + /// Moves each time the terminal publishes a route ([`InputPort::route_changed`]) or will install none + /// ([`InputPort::reconnect_ended`]): ends the writer's reconnect wait. route_epoch: u64, /// Tests: the writer waits here before it writes each frame. #[cfg(test)] @@ -314,6 +315,17 @@ impl InputPort { self.shared.changed.notify_all(); } + /// The terminal will install no host connection for now (its reader ended or gave the host up, or its owner + /// detaches it): a writer waiting for a reconnect checks again at once and fails its frame, as the old path's + /// write failed then (#35082 review 2). + pub(crate) fn reconnect_ended(&self) { + let mut state = self.shared.state.lock().unwrap_or_else(PoisonError::into_inner); + state.route_epoch = state.route_epoch.wrapping_add(1); + state.reconnecting = false; + drop(state); + self.shared.changed.notify_all(); + } + fn enqueue_frame( &self, terminal: &Arc, @@ -517,8 +529,9 @@ fn run_writer(shared: &PortShared, terminal: &Weak) { } } -/// Waits until the terminal publishes a route after the one seen at epoch `observed` ([`InputPort::route_changed`]), -/// until the port closes or until `deadline`; true when a new route came. Marks the writer waiting for a reconnect +/// Waits until the terminal publishes a route after the one seen at epoch `observed` ([`InputPort::route_changed`]) +/// or ends its reconnect ([`InputPort::reconnect_ended`]), until the port closes or until `deadline`; true when the +/// epoch moved (the caller checks the route and the terminal's state again). Marks the writer waiting for a reconnect /// meanwhile, so barriers do not wait for it (a new route clears the mark at once). fn wait_for_route(shared: &PortShared, observed: u64, deadline: Instant) -> bool { let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/shutdown.rs b/cmux-tui/crates/cmux-tui-core/src/surface/shutdown.rs index 691abd68fdc9..81d3bc1d0f50 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/shutdown.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/shutdown.rs @@ -134,6 +134,9 @@ impl Surface { // racing termination by reconnecting a Surface that no longer // exists in the mux topology. pty.owner_detaching.store(true, Ordering::Release); + // A port writer waiting for a reconnect fails its frame now (#35082 review 2); only the port + // lock, which no holder of the runtime lock waits for. + pty.input_port.reconnect_ended(); { let mut runtime = pty.runtime.lock().unwrap(); match &mut *runtime { @@ -175,6 +178,9 @@ impl Surface { Surface::Pty(pty) => { if let PtyRuntime::Hosted(host) = &*pty.runtime.lock().unwrap() { pty.owner_detaching.store(true, Ordering::Release); + // A port writer waiting for a reconnect fails its frame now (#35082 review 2); only the port + // lock, which no holder of the runtime lock waits for. + pty.input_port.reconnect_ended(); host.disconnect(); return; } @@ -187,6 +193,9 @@ impl Surface { &*pty.runtime.lock().unwrap() { pty.owner_detaching.store(true, Ordering::Release); + // A port writer waiting for a reconnect fails its frame now (#35082 review 2); only the port + // lock, which no holder of the runtime lock waits for. + pty.input_port.reconnect_ended(); holder.detach(); return; } @@ -208,6 +217,9 @@ impl Surface { let runtime = pty.runtime.lock().unwrap(); if let PtyRuntime::Hosted(host) = &*runtime { pty.owner_detaching.store(true, Ordering::Release); + // A port writer waiting for a reconnect fails its frame now (#35082 review 2); only the port + // lock, which no holder of the runtime lock waits for. + pty.input_port.reconnect_ended(); let mut gap = None; if host.supports_journal_detach_fence() && let Err(error) = host.detach_for_daemon_shutdown_until(deadline) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs index a3ff8149a633..199dc9f00699 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface/terminal_runtime.rs @@ -83,6 +83,8 @@ impl PtyTerminalRuntime { /// terminal): input that waits out a lost connection fails now, as the old path's write failed then. pub(super) fn reader_gave_up(&self) { self.stream_progress.notify(); + // The input port's writer waits on its own route epoch, not on the stream (#35082 review 2). + self.input_port.reconnect_ended(); } /// The reader shut the current host connection down and keeps retrying (`Failed`): input is refused until a